curvefs/client: fix the problem that chunkCacheManager has been released in advance during the Release process of dataCache

previously, s3 compact task stores a raw pointer of copyset,
so after copyset is been purged, s3 compact task will cause
curvefs-metaserver process exited.

now, let s3 compact task store a shared copyset node.

.bazelrc

@@ -1,10 +0,0 @@
-build --verbose_failures
-
-build --define=with_glog=true --define=libunwind=true
-build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
-build --cxxopt -Wno-error=format-security
-build:gcc7-later --cxxopt -faligned-new
-build --incompatible_blacklisted_protos_requires_proto_info=false
-build --copt=-fdiagnostics-color=always
-
-run --copt=-fdiagnostics-color=always

.bazelversion

@@ -1 +1 @@
-4.2.2
+0.17.2

.clang-format

@@ -1,147 +0,0 @@
----
-Language:        Cpp
-AccessModifierOffset: -3
-AlignAfterOpenBracket: Align
-AlignConsecutiveMacros: true
-AlignConsecutiveAssignments: false
-AlignConsecutiveBitFields: false
-AlignConsecutiveDeclarations: false
-AlignEscapedNewlines: Right
-AlignOperands:   Align
-AlignTrailingComments: true
-AllowAllArgumentsOnNextLine: true
-AllowAllConstructorInitializersOnNextLine: true
-AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortEnumsOnASingleLine: true
-AllowShortBlocksOnASingleLine: Never
-AllowShortCaseLabelsOnASingleLine: false
-AllowShortFunctionsOnASingleLine: All
-AllowShortLambdasOnASingleLine: All
-AllowShortIfStatementsOnASingleLine: Never
-AllowShortLoopsOnASingleLine: false
-AlwaysBreakAfterDefinitionReturnType: None
-AlwaysBreakAfterReturnType: None
-AlwaysBreakBeforeMultilineStrings: false
-AlwaysBreakTemplateDeclarations: MultiLine
-BinPackArguments: true
-BinPackParameters: true
-BraceWrapping:
-  AfterCaseLabel:  false
-  AfterClass:      false
-  AfterControlStatement: Never
-  AfterEnum:       false
-  AfterFunction:   false
-  AfterNamespace:  false
-  AfterObjCDeclaration: false
-  AfterStruct:     false
-  AfterUnion:      false
-  AfterExternBlock: false
-  BeforeCatch:     false
-  BeforeElse:      false
-  BeforeLambdaBody: false
-  BeforeWhile:     false
-  IndentBraces:    false
-  SplitEmptyFunction: true
-  SplitEmptyRecord: true
-  SplitEmptyNamespace: true
-BreakBeforeBinaryOperators: None
-BreakBeforeBraces: Attach
-BreakBeforeInheritanceComma: false
-BreakInheritanceList: BeforeColon
-BreakBeforeTernaryOperators: true
-BreakConstructorInitializersBeforeComma: false
-BreakConstructorInitializers: BeforeColon
-BreakAfterJavaFieldAnnotations: false
-BreakStringLiterals: true
-ColumnLimit:     80
-CommentPragmas:  '^ IWYU pragma:'
-CompactNamespaces: false
-ConstructorInitializerAllOnOneLineOrOnePerLine: false
-ConstructorInitializerIndentWidth: 4
-ContinuationIndentWidth: 4
-Cpp11BracedListStyle: true
-DeriveLineEnding: true
-DerivePointerAlignment: false
-DisableFormat:   false
-ExperimentalAutoDetectBinPacking: false
-FixNamespaceComments: true
-ForEachMacros:
-  - foreach
-  - Q_FOREACH
-  - BOOST_FOREACH
-IncludeBlocks:   Preserve
-IncludeCategories:
-  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
-    Priority:        2
-    SortPriority:    0
-  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
-    Priority:        3
-    SortPriority:    0
-  - Regex:           '.*'
-    Priority:        1
-    SortPriority:    0
-IncludeIsMainRegex: '(Test)?$'
-IncludeIsMainSourceRegex: ''
-IndentCaseLabels: false
-IndentCaseBlocks: false
-IndentGotoLabels: true
-IndentPPDirectives: None
-IndentExternBlock: AfterExternBlock
-IndentWidth:     4
-IndentWrappedFunctionNames: false
-InsertTrailingCommas: None
-JavaScriptQuotes: Leave
-JavaScriptWrapImports: true
-KeepEmptyLinesAtTheStartOfBlocks: true
-MacroBlockBegin: ''
-MacroBlockEnd:   ''
-MaxEmptyLinesToKeep: 2
-NamespaceIndentation: None
-ObjCBinPackProtocolList: Auto
-ObjCBlockIndentWidth: 2
-ObjCBreakBeforeNestedBlockParam: true
-ObjCSpaceAfterProperty: false
-ObjCSpaceBeforeProtocolList: true
-PenaltyBreakAssignment: 2
-PenaltyBreakBeforeFirstCallParameter: 19
-PenaltyBreakComment: 300
-PenaltyBreakFirstLessLess: 120
-PenaltyBreakString: 1000
-PenaltyBreakTemplateDeclaration: 10
-PenaltyExcessCharacter: 1000000
-PenaltyReturnTypeOnItsOwnLine: 60
-PointerAlignment: Right
-ReflowComments:  true
-SortIncludes:    false
-SortUsingDeclarations: true
-SpaceAfterCStyleCast: false
-SpaceAfterLogicalNot: false
-SpaceAfterTemplateKeyword: true
-SpaceBeforeAssignmentOperators: true
-SpaceBeforeCpp11BracedList: false
-SpaceBeforeCtorInitializerColon: true
-SpaceBeforeInheritanceColon: true
-SpaceBeforeParens: ControlStatements
-SpaceBeforeRangeBasedForLoopColon: true
-SpaceInEmptyBlock: false
-SpaceInEmptyParentheses: false
-SpacesBeforeTrailingComments: 2
-SpacesInAngles:  false
-SpacesInConditionalStatement: false
-SpacesInContainerLiterals: true
-SpacesInCStyleCastParentheses: false
-SpacesInParentheses: false
-SpacesInSquareBrackets: false
-SpaceBeforeSquareBrackets: false
-Standard:        Cpp11
-StatementMacros:
-  - Q_UNUSED
-  - QT_REQUIRE_VERSION
-TabWidth:        4
-UseCRLF:         false
-UseTab:          Never
-WhitespaceSensitiveMacros:
-  - STRINGIZE
-  - PP_STRINGIZE
-  - BOOST_PP_STRINGIZE
-...

.clangd

@@ -1,12 +0,0 @@
-Diagnostics:
-  ClangTidy:
-    Add: [performance-*, modernize-*, readability-*,bugprone-*,]
-    Remove: [modernize-use-trailing-return-type]
-    CheckOptions:
-      readability-identifier-naming.VariableCase: camelBack
-  UnusedIncludes: Strict
-
-InlayHints:
-  Enabled: No
-  ParameterNames: No
-  DeducedTypes: No

.github/ISSUE_TEMPLATE/bug_report.md

@@ -19,7 +19,11 @@ assignees: ''
 **Versions (各种版本)**
 OS:
 Compiler:
-branch:
-commit id:
+curve-mds:
+curve-chunkserver:
+curve-snapshotcloneserver:
+curve-sdk:
+nebd:
+curve-nbd:
 
 **Additional context/screenshots (更多上下文/截图)**

.github/pull_request_template.md

@@ -2,7 +2,7 @@
 
 ### What problem does this PR solve?
 
-Issue Number: #xxx <!-- replace xxx with issue number -->
+Issue Number: close #xxx <!-- REMOVE this line if no issue to close -->
 
 Problem Summary:
 
@@ -12,7 +12,7 @@ What's Changed:
 
 How it Works:
 
-Side effects(Breaking backward compatibility? Performance regression?):
+Side effects(Breaking backward compatibility? Performance regression?): 
 
 ### Check List
 

.github/workflows/issue-action.yml

@@ -1,16 +0,0 @@
-name: Issue assignment
-
-on:
-    issues:
-        types: [opened]
-
-jobs:
-    auto-assign:
-        runs-on: ubuntu-latest
-        steps:
-            - name: 'Auto-assign issue'
-              uses: pozil/auto-assign-issue@v1.10.1
-              with:
-                  assignees: cw123,ilixiaocui,wuhongsong,Cyber-SiKu,jolly-sy
-                  numOfAssignee: 1
-                  allowSelfAssign: true

.gitignore

@@ -35,7 +35,6 @@ CMakeLists.txt
 bin
 *.temp
 .clwb
-*.swp
 
 # cscope
 cscope.out
@@ -74,11 +73,7 @@ bazel-testlogs
 *.log
 runlog/
 
-# history
-.gdb_history
-
 .clang-format
-.clangd
 !curve-snapshotcloneserver-nginx/app/lib
 !nebd/nebd-package/usr/bin
 
@@ -91,7 +86,6 @@ test/client/configs/*
 projects/*
 docker/curvebs
 docker/base/*
-docker/debian*/*so*
 !docker/base/Dockerfile
 !docker/base/Makefile
 
@@ -103,66 +97,9 @@ curvefs/BUILD_MODE
 *.pyc
 .facts/
 *retry
-# monitor
-curvefs/monitor/prometheus/target.json
 
-curvefs/docker/*/curvefs
 curvefs/docker/curvefs
 curvefs/docker/base/*
-curvefs/docker/debian*/*so*
 !curvefs/docker/base/Dockerfile
 !curvefs/docker/base/Makefile
 curvefs/BUILD_MODE
-
-.BUILD_MODE
-__not_found__
-
-# thirdparties
-thirdparties/rocksdb/lib/
-thirdparties/rocksdb/include/
-thirdparties/rocksdb/rocksdb/
-thirdparties/rocksdb/*log
-thirdparties/rocksdb/*.tar.gz
-thirdparties/aws/*.tar.gz
-thirdparties/etcdclient/tmp/
-thirdparties/etcdclient/*.h
-thirdparties/memcache/*.tar.gz
-thirdparties/memcache/libmemcached-*/
-
-/external
-/bazel-*
-/compile_commands.json
-/.cache/
-
-docker/curvebs
-docker/*/curvebs
-
-curvefs/docker/curvefs
-curvefs/docker/*/curvefs
-storage_*
-
-tools-v2/sbin/*
-tools-v2/proto/proto/*
-tools-v2/proto/curvefs/*
-tools-v2/*/*.test
-tools-v2/__debug_bin
-tools-v2/vendor/
-
-.test
-.note
-.playground
-.dumpfile
-metastore_test.dat
-GPATH
-GRTAGS
-GTAGS
-core.*
-
-test/integration/*.conf
-test/integration/client/config/client.conf*
-test/integration/snapshotcloneserver/config/*.conf
-
-.pre-commit-config.yaml
-
-*.deb
-*.whl

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "nbd"]
-	path = nbd
-	url = https://github.com/opencurve/curve-nbd

.obm.cfg

@@ -1,2 +0,0 @@
-container_name: curve-build-playground-master
-container_image: opencurvedocker/curve-base:build-debian9

CHANGELOG-2.1.md

@@ -1,41 +0,0 @@
-# CHANGELOG of v2.1
-
-## new features
-
-- [CurveFS: a curve filesystem can be mounted by multi fuse clients.](https://github.com/opencurve/curve/pull/1101)
-
-- [CurveFS: support summary info in dir xattr.](https://github.com/opencurve/curve/pull/1150)
-
-- [CurveFS: support Multiple s3.](https://github.com/opencurve/curve/pull/1132)
-
-## optimization
-
-- [CurveFS client: adapter lru list for disk cache.](https://github.com/opencurve/curve/pull/1088)
-
-- [CurveFS: meta balance.](https://github.com/opencurve/curve/pull/1105)
-
-- [CurveFS metaserver: let s3 compact task store a shared copyset node.](https://github.com/opencurve/curve/pull/1165)
-
-- [CurveFS: add metric for mds topology to update metaserver metric.](https://github.com/opencurve/curve/pull/1177)
-
-## bug fix
-
-- [CurveFS client: fix bug of getleader always fails causes stack overflow.](https://github.com/opencurve/curve/pull/1070)
-
-- [CurveFS tool: fix copyset health check error.](https://github.com/opencurve/curve/pull/1024)
-
-- [CurveFS client: fix client release read data cache core dump.](https://github.com/opencurve/curve/pull/1090)
-
-- [CurveBS nbd: fix misspell in log.](https://github.com/opencurve/curve/pull/1073)
-
-- [CurveFS mds: update copyset condidate.](https://github.com/opencurve/curve/pull/1085)
-
-- [CurveFS client: fix bug of rpc overtime time not backoff.](https://github.com/opencurve/curve/pull/1139)
-
-- [CurveFS client: fix bug when upload failed.](https://github.com/opencurve/curve/pull/1130)
-
-- [CurveFS client: fix the problem that chunkCacheManager has been releaed.](https://github.com/opencurve/curve/pull/1179)
-
-- [CurveFS: fix partition num in topology metric.](https://github.com/opencurve/curve/pull/1187)
-
-- [CurveFS client: fix core dump of DataCache::Flush & fix rpc timeout not backoff.](https://github.com/opencurve/curve/pull/1193)

CHANGELOG-2.2.md

@@ -1,23 +0,0 @@
-# CHANGELOG of v2.2
-
-## Feature
-
-- [curvefs: support curvebs as data backend](https://github.com/opencurve/curve/pull/1207)
-
-- [curvefs/metaserver: now we support rocksdb storage](https://github.com/opencurve/curve/pull/1214)
-
-## Optimization
-
-- [optimize copyset creation](https://github.com/opencurve/curve/pull/1211)
-
-- [curvefs/client: s3 adaptor unit test optimization](https://github.com/opencurve/curve/pull/1227)
-
-- [curvefs: xattr summary info support hard link](https://github.com/opencurve/curve/pull/1185)
-
-## Bugfix
-
-- [curvefs/client: fix waitIntervalSec_ is not init](https://github.com/opencurve/curve/pull/1248)
-
-- [metaserver: fix empty configuration after load snapshot](https://github.com/opencurve/curve/pull/1260)
-
-- [curvefs-mds: fix an uninitialized that spawn infinite threads](https://github.com/opencurve/curve/pull/1272)

CHANGELOG-2.3.md

@@ -1,126 +0,0 @@
-# CHANGELOG of v2.3
-
-Previous change logs can be found at [CHANGELOG-2.2](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.2.md)
-
-## Notable Changes
-
-- [CurveFS : adapt to curveadm to support curvebs as data backend](https://github.com/opencurve/curve/pull/1349)
-
-- [CurveFS monitor: promethus](https://github.com/opencurve/curve/pull/1237)
-
-- [CurveFS client: perf optimize](https://github.com/opencurve/curve/pull/1194)
-
-- [CurveFS metaserver: fixed rocksdb storage memory leak caused by unreleasing iterator and transaction.](https://github.com/opencurve/curve/pull/1388)
-
-## new features
-
-- [CurveFS : adapt to curveadm to support curvebs as data backend](https://github.com/opencurve/curve/pull/1349)
-
-- [CurveFS monitor: promethus](https://github.com/opencurve/curve/pull/1237)
-
-- [CurveFS client: add s3.useVirtualAddressing config, default value: false](https://github.com/opencurve/curve/pull/1253)
-
-## optimization
-
-- [CurveFS metaserver: speed up getting inode by padding inode's s3chunk](https://github.com/opencurve/curve/pull/1344)
-
-- [CurveFS client: perf optimize](https://github.com/opencurve/curve/pull/1194)
-
-## Data Performance
-Hardware: 3 nodes (3*mds, 9*metaserver), each with:
- - Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GHz
- - 256G RAM
- - disk cache: INTEL SSDSC2BB80 800G(iops is about 30000+,bw is about 300MB)
-
-  - performance is as follows:
-
-s3 backend is minio and the cto is disable([what is cto](https://github.com/opencurve/curve/blob/master/docs/cn/CurveFS%E6%94%AF%E6%8C%81%E5%A4%9A%E6%8C%82%E8%BD%BD.pdf)). and as you know, the performance of read may associated with read cache hit.
-
-| minio + diskcache(free)| iops/bandwidth | avg-latency(ms) | clat 99.00th (ms) | clat 99.99th (ms) |
-| :----: | :----: | :----: | :----: | :----: |
-| (numjobs 1) (50G filesize) 4k randwrite | 3539 | 0.281 | 1.5 | 16 |
-| (numjobs 1) (50G filesize) 4k randread | 2785 | 0.357 | 0.9 | 5.8|
-| (numjobs 1) (50G filesize) 512k write | 290 MB/s | 1 | 600 ms | 248|
-| (numjobs 1) (50G filesize) 512k read | 216 MB/s | 4.3 | 275 ms | 7.6 |
-
-
-| minio + diskcache(near full)| iops/bandwidth | avg-latency(ms) | clat 99.00th (ms) | clat 99.99th (ms) |
-| :----: | :----: | :----: | :----: | :----: |
-| (numjobs 1) (50G filesize) 4k randwrite | 2988 | 0.3 | 1.2 | 18 |
-| (numjobs 1) (50G filesize) 4k randread | 1559 | 0.6 | 1.9 | 346|
-| (numjobs 1) (50G filesize) 512k write | 266 MB/s | 0.9| 600 ms | 396|
-| (numjobs 1) (50G filesize) 512k read | 82 MB/s | 86 | 275 ms | 901|
-
-
-| minio + diskcache(full)| iops/bandwidth | avg-latency(ms) | clat 99.00th (ms) | clat 99.99th (ms) |
-| :----: | :----: | :----: | :----: | :----: |
-| (numjobs 1) (20G filesize * 5) 4k randwrite | 2860 | 1.7| 14 | 41 |
-| (numjobs 1) (20G filesize * 5) 4k randread | 76 | 65 | 278 | 725|
-| (numjobs 1) (20G filesize * 5) 512k write | 240 MB/s | 10| 278 | 513|
-| (numjobs 1) (20G filesize * 5) 512k read | 192 MB/s | 12 | 40 | 1955 |
-
-## Metadata Performance
-
-Cluster Topology: 3 nodes (3mds, 6metaserver), each metaserver is deployed on a separate SATA SSD
-
-Configuration: use default configuration
-
-Test tool: mdtest v3.4.0
-
-Test cases:
-- case 1: the directory structure is relatively flat and total 130,000 dirs and files.
-mdtest -z 2 -b 3 -I 10000 -d /mountpoint
-
-- case 2: the directory structure is relatively deep and total 204,700 dirs and files.
-mdtest -z 10 -b 2 -I 100 -d /mountpoint
-
-
-| Case | Dir creation | Dir stat | Dir rename | Dir removal | File creation | File stat | File read | File removal | Tree creation | Tree removal |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| case 1 | 1320 | 5853 | 149 | 670 | 1103 | 5858 | 1669 | 1419 | 851 | 64 |
-| case 2 | 1283 | 5205 | 147 | 924 | 1081 | 5316 | 1634 | 1260 | 1302 | 887 |
-
-You can set configuration item ‘fuseClient.enableMultiMountPointRename’ to false in client.conf if you don't need concurrent renames on multiple mountpoints on the same filesystem. It will improve the performance of metadata.
-
-fuseClient.enableMultiMountPointRename=false
-
-| Case | Dir creation | Dir stat | Dir rename | Dir removal | File creation | File stat | File read | File removal | Tree creation | Tree removal |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| case 1 | 1537 | 7885 | 530 | 611 | 1256 | 7998 | 1861 | 1614 | 1050 | 72 |
-| case 2 | 1471 | 6328 | 509 | 1055 | 1237 | 6874 | 1818 | 1454 | 1489 | 1034 |
-
-## bug fix
-
-- [CurveFS mds: fix an uninitialized that spawn infinite threads](https://github.com/opencurve/curve/pull/1270)
-
-- [CurveFS client: fix missing update extent cache](https://github.com/opencurve/curve/pull/1279)
-
-- [CurveFS: fix a compatible issue which caused by modify FsFileType ](https://github.com/opencurve/curve/pull/1359)
-
-- [CurveFS client: fix bug cachedisk never trim](https://github.com/opencurve/curve/pull/1378)
-
-- [CurveFS client: fix bug of io hang](https://github.com/opencurve/curve/pull/1377)
-
-- [CurveFS metaserver: fixed rocksdb storage memory leak caused by unreleasing iterator and transaction.](https://github.com/opencurve/curve/pull/1388)
-
-- [CurveFS client: Adjust the number of retries to the maximum to avoid an error when the number of retries is reached](https://github.com/opencurve/curve/pull/1410)
-
-- [CurveFS : fix clear copyset creating flag when create copyset success](https://github.com/opencurve/curve/pull/1417)
-
-- [CurveFS mds: fix miss set txid when create partition and fix inodeId type](https://github.com/opencurve/curve/pull/1479)
-
-- [CurveFS client: fix the data iteration error when rpc retry.](https://github.com/opencurve/curve/pull/1474)
-
-- [CurveFS metaserver: skip find dentry when loading from snapshot](https://github.com/opencurve/curve/pull/1460)
-
-- [CurveFS : fix compile error which was caused by LatencyUpdater success](https://github.com/opencurve/curve/pull/1508)
-
-- [CurveFS mds: fix create partition error at parallel case](https://github.com/opencurve/curve/pull/1511)
-
-- [CurveFS metaserver: fix the data iteration error when rpc retry.metaserver: recover s3ChunkInfoRemove field for GetOrModify](https://github.com/opencurve/curve/pull/1404)
-
-- [CurveFS metaserver: fixed s3chunkinfo was padding into inode when it wasnt needed](https://github.com/opencurve/curve/pull/1510)
-
-- [Curve common: fix timer bug](https://github.com/opencurve/curve/pull/1492)
-
-- [CurveFS client: fix statfs problem](https://github.com/opencurve/curve/pull/1620)

CHANGELOG-2.4.md

@@ -1,47 +0,0 @@
-# CHANGELOG of v2.4
-
-Previous change logs can be found at [CHANGELOG-2.3](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.3.md)
-
-## Notable Changes
-
-- [Update aws-sdk-cpp version to 1.9](https://github.com/opencurve/curve/pull/1780)
-
-- [add feature of warmup](https://github.com/opencurve/curve/pull/1793)
-
-
-## new features
-
-- [CurveFS: implement setxattr interface](https://github.com/opencurve/curve/pull/1935)
-
-- [update aws-sdk-cpp: add ip and custom port support](https://github.com/opencurve/curve/pull/1795)
-
-- [CurveFS client: add s3.useVirtualAddressing config, default value: false](https://github.com/opencurve/curve/pull/1253)
-
-- [Update aws-sdk-cpp version to 1.9](https://github.com/opencurve/curve/pull/1780)
-
-- [add feature of warmup](https://github.com/opencurve/curve/pull/1793)
-
-## optimization
-
-- [CurveFS: update attr and extent in single rpc](https://github.com/opencurve/curve/pull/1784)
-
-- [copyset schedule select copyset random](https://github.com/opencurve/curve/pull/1811)
-
-- [CurveFS: only update dirty inode metadata](https://github.com/opencurve/curve/pull/1853)
-
-
-## bug fix
-
-- [CurveFS:  fix RefreshInode do not refresh when inode exist](https://github.com/opencurve/curve/pull/1800)
-
-- [fix misuse Locked & Unlocked calling contract](https://github.com/opencurve/curve/pull/1279)
-
-- [make callback of s3async request async to avoid deadlock](https://github.com/opencurve/curve/pull/1854)
-
-- [CurveFS :fix update nlink error ](https://github.com/opencurve/curve/pull/1901)
-
-- [CurveFS client: fix umount bug](https://github.com/opencurve/curve/pull/1923)
-
-- [CurveFS client:  fix refresh inode will overwrite data in cache when enabel cto.](https://github.com/opencurve/curve/pull/1993)
-
-

CHANGELOG-2.5.md

@@ -1,49 +0,0 @@
-# CHANGELOG of v2.5
-
-Previous change logs can be found at [CHANGELOG-2.4](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.4.md)
-
-
-
-## New Features
-
-- Memcache support
-    - [curvefs/client: support memcached cluster](https://github.com/opencurve/curve/pull/2096) @ilixiaocui
-    - [curvefs/mds: support memcache cluster](https://github.com/opencurve/curve/pull/2108) @Cyber-SiKu
-    - [curvefs/client: add global cache client like memcached](https://github.com/opencurve/curve/pull/2102) @fansehep
-    - [curvefs/client: fix client core dump error](https://github.com/opencurve/curve/pull/2157) @ilixiaocui
-    - [add ut](https://github.com/opencurve/curve/pull/2164) @Cyber-SiKu
-
-- curvefs new tools 
-    - [add delete impl](https://github.com/opencurve/curve/pull/2088) @shentupenghui
-    - [feat: [tools-v2] bs list dir](https://github.com/opencurve/curve/pull/2082) @Sindweller
-    - [fix missing docs links](https://github.com/opencurve/curve/pull/2082) @zyb521
-    - [curve/toos-v2: add list client #2037](https://github.com/opencurve/curve/pull/2076) @tsonglew
-    - [support create volume directory by curve_ops_tool](https://github.com/opencurve/curve/pull/2078) @aspirer
-
-- [install: added playground.](https://github.com/opencurve/curve/pull/2053) @Wine93
-
-## Optimization
-
-- [update the braft to vesion v1.1.2](https://github.com/opencurve/curve/pull/2091) @tangwz
-
-- [curvefs/client: local cache policy optimization](https://github.com/opencurve/curve/pull/2064) @Tangruilin
-
-
-- [sync by threadpool](https://github.com/opencurve/curve/pull/1912) @fansehep
-
-- Merge block storage and file storage compilation scripts
-    - [Merge block storage and file storage compilation scripts
-](https://github.com/opencurve/curve/pull/2089) @linshiyx
-    - [Add help_msg in Makefile](https://github.com/opencurve/curve/pull/2133) @linshiyx
-    - [fix image.sh](https://github.com/opencurve/curve/pull/2124) @SeanHai
-
-- [fix bs list zone typo](https://github.com/opencurve/curve/pull/2066) @tsonglew 
-
-- [spell optimize](https://github.com/opencurve/curve/pull/2059) @shentupenghui
-
-
-## Bug Fix
-
-- aws s3 sdk revert
-    - [revert new s3 sdk due to critical bug](https://github.com/opencurve/curve/pull/2149) @h0hmj
-    - [curvefs/client: change the default value of s3](https://github.com/opencurve/curve/pull/2158) @wuhongsong

LICENSE

@@ -1,202 +1,156 @@
+Files: *
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
+Files: curve-snapshotcloneserver-nginx/docker/Dockerfile
+Copyright: Copyright (c) 2017-2020, Evan Wies evan@neomantra.net.
+License:
+================================================================================
+docker-openresty is licensed under the 2-clause BSD license.
 
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+Copyright (c) 2017-2020, Evan Wies evan@neomantra.net.
 
-   1. Definitions.
+This module is licensed under the terms of the BSD license.
 
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+================================================================================
 
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
+Files: curve-snapshotcloneserver-nginx/app/etc/config.lua
+       curve-snapshotcloneserver-nginx/app/lib/resty/argutils.lua
+       curve-snapshotcloneserver-nginx/app/lib/resty/checkups.lua
+       curve-snapshotcloneserver-nginx/app/lib/resty/checkups/*
+       curve-snapshotcloneserver-nginx/app/src/modules/httpipe.lua
+       curve-snapshotcloneserver-nginx/app/src/modules/httproxy.lua
+       curve-snapshotcloneserver-nginx/app/src/modules/reqlimit.lua
+Copyright: Copyright (C) 2014-2016 UPYUN, Inc.
+License:
+================================================================================
+The bundle itself is licensed under the 2-clause BSD license.
 
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
+Copyright (c) 2016, UPYUN(又拍云) Inc.
 
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
+This module is licensed under the terms of the BSD license.
 
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+================================================================================
 
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
+Files: curve-snapshotcloneserver-nginx/app/lib/resty/uuid.lua
+Copyright: Copyright (c) 2016-2019 Thibault Charbonnier
+License:
+================================================================================
+The MIT License (MIT)
 
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
+    Copyright (c) 2016-2019 Thibault Charbonnier
 
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
 
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
 
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+================================================================================
 
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
+Files: monitor/*
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: GPL 2.0 (see LICENSE_GPL)
 
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
+Files: src/chunkserver/raftsnapshot/curve_file_service.cpp
+       src/chunkserver/raftsnapshot/curve_file_service.h
+       src/chunkserver/raftsnapshot/curve_snapshot.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot.h
+       src/chunkserver/raftsnapshot/curve_snapshot_copier.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_copier.h
+       src/chunkserver/raftsnapshot/curve_snapshot_file_reader.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_file_reader.h
+       src/chunkserver/raftsnapshot/curve_snapshot_reader.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_reader.h
+       src/chunkserver/raftsnapshot/curve_snapshot_storage.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_storage.h
+       src/chunkserver/raftsnapshot/curve_snapshot_writer.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_writer.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015 Baidu.com, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
+Files: src/common/authenticator.cpp
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2016 Baidu, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
+Files: src/common/hash.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015, Baidu.com
+           Copyright (c) 2011 The LevelDB Authors
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
+Files: src/common/string_util.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015, Baidu.com
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
+Files: src/mds/nameserver2/file_lock.cpp
+       src/mds/nameserver2/file_lock.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2016, Baidu.com, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
+Files: src/tools/nbd/*
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: GPL 2.0 (see LICENSE_GPL)
 
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
+Files: src/tools/nbd/nbd-netlink.h
+Copyright: Copyright (C) 2017 Facebook
+License: GPL 2.0 (see LICENSE_GPL)
 
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
+Files: src/tools/nbd/define.h
+       src/tools/nbd/NBDController.cpp
+       src/tools/nbd/NBDController.h
+       src/tools/nbd/NBDServer.cpp
+       src/tools/nbd/NBDServer.h
+       src/tools/nbd/util.cpp
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (C) 2015 - 2016 Kylin Corporation
+License: LGPL 2.1 (see LICENSE_LGPL)
 
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
+Files: src/tools/nbd/texttable.cpp
+       src/tools/nbd/texttable.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (C) 2012 Inktank Storage, Inc.
+License: LGPL 2.1 (see LICENSE_LGPL)
 
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
+Files: test/chunkserver/raftsnapshot/curve_snapshot_storage_test.cpp
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015 Baidu.com, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-   END OF TERMS AND CONDITIONS
+Files: test/tools/nbd/*
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: GPL 2.0 (see LICENSE_GPL)
 
-   APPENDIX: How to apply the Apache License to your work.
+Files: thirdparties/aws/*
+       common.bzl
+Copyright: Copyright 2019 The TensorFlow Authors
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Files: tools/ci/filterbr.py
+       tools/ci/gen-coverage.py
+License: GPL 3.0 (see LICENSE_GPLv3)

LICENSE_APACHE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE_GPL

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

LICENSE_GPLv3

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

LICENSE_LGPL

@@ -0,0 +1,502 @@
+          GNU LESSER GENERAL PUBLIC LICENSE
+               Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard.  To achieve this, non-free programs must be
+allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+          GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+             END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!

MAINTAINERS

@@ -0,0 +1,13 @@
+# The list of current Curve maintainers and reviewers:
+
+# Names should be added to this file like so:
+#     Name, <email address>, @GITHUB_HANDLE
+
+Wang Pan, <hzwangpan@corp.netease.com>, @aspirer
+Li Xiaocui, <lixiaocui1@corp.netease.com>, @ilixiaocui
+Chen Wei, <hzchenwei7@corp.netease.com>, @cw123
+Xu Chaojie, <xuchaojie@corp.netease.com>, @xu-chaojie
+Wu Hanqing, <wuhanqing@corp.netease.com>, @wu-hanqing
+Chen Yunhui, <chenyunhui@corp.netease.com>, @YunhuiChen
+Wang Hai, <wanghai01@corp.netease.com>, @SeanHai
+Hu Yao, <huyao@corp.netease.com>, @baijiaruo

MAINTAINERS.md

@@ -1,7 +0,0 @@
-# Maintainers
-
-The current maintainers of the Curve project are:
-
-* Pan WANG, [aspirer](https://github.com/aspirer), <aspirer2004@gmail.com>, core maintainer
-* XiaoCui Li, [ilixiaocui](https://github.com/ilixiaocui), <ilixiaocui@163.com>, core maintainer
-* opencurveadmin, [opencurveadmin](https://github.com/opencurveadmin), <hzchenwei7@corp.netease.com>, project management

Makefile

@@ -1,81 +1,21 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image playground check test
+.PHONY: list build install image
 
-stor?=""
 prefix?= "$(PWD)/projects"
 release?= 0
-dep?= 0
 only?= "*"
 tag?= "curvebs:unknown"
 case?= "*"
-os?= "debian9"
-ci?=0
-
-define help_msg
-## list
-Usage:
-    make list stor=bs/fs
-Examples:
-    make list stor=bs
-
-## build
-Usage:
-    make build stor=bs/fs only=TARGET dep=0/1 release=0/1 os=OS
-Examples:
-    make build stor=bs only=//src/chunkserver:chunkserver
-    make build stor=bs only=src/* dep=0
-    make build stor=fs only=test/* os=debian9
-    make build stor=fs release=1
-
-
-## dep
-Usage:
-    make dep stor=bs/fs
-Examples:
-    make dep stor=bs
-
-
-## install
-Usage:
-    make install stor=bs/fs prefix=PREFIX only=TARGET
-Examples:
-    make install stor=bs prefix=/usr/local/curvebs only=*
-    make install stor=bs prefix=/usr/local/curvebs only=chunkserver
-    make install stor=fs prefix=/usr/local/curvefs only=etcd
-
-
-## image
-Usage:
-    make image stor=bs/fs tag=TAG os=OS
-Examples:
-    make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian9
-endef
-export help_msg
-
-help:
-	@echo "$$help_msg"
 
 list:
-	@bash util/build.sh --stor=$(stor) --list
+	@bash util/build.sh --list
 
 build:
-	@bash util/build.sh --stor=${stor} --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os)
-
-dep:
-	@bash util/build.sh --stor=$(stor) --only="" --dep=1
+	@bash util/build.sh --only=$(only) --release=$(release)
 
 install:
-	@bash util/install.sh --stor=$(stor) --prefix=$(prefix) --only=$(only)
+	@bash util/install.sh --prefix=$(prefix) --only=$(only)
 
 image:
-	@bash util/image.sh $(stor) $(tag) $(os)
-
-playground:
-	@bash util/playground.sh
-
-check:
-	@bash util/check.sh $(stor)
-
-test:
-	@bash util/test.sh $(stor) $(only)
+	@bash util/image.sh $(tag)

README.md

@@ -1,180 +1,51 @@
+[中文版](README_cn.md)
 
 
-<div align=center> <img src="docs/images/curve-logo1-nobg.png" width = 45%>
+<img src="docs/images/curve-logo1.png"/>
 
-<div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
+# CURVE
 
-**A cloud-native distributed storage system**
+[![Jenkins Coverage](https://img.shields.io/jenkins/coverage/cobertura?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_untest_job%2F)](http://59.111.91.248:8080/job/curve_untest_job/HTML_20Report/)
+[![Robot failover](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_failover_testjob%2F&label=failover)](http://59.111.91.248:8080/job/curve_failover_testjob/)
+[![Robot interface](https://img.shields.io/jenkins/tests?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_robot_job%2F)](http://59.111.91.248:8080/job/curve_robot_job/)
+[![Curve_choas](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_choas_test%2F&label=choas)](http://59.111.91.248:8080/job/curve_choas_test/)
+[![BUILD Status](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fopencurve_multijob%2F)](http://59.111.91.248:8080/job/opencurve_multijob/lastBuild)
+[![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://github.com/opencurve/curve/tree/master/docs)
+[![Releases](https://img.shields.io/github/v/release/opencurve/curve?include_prereleases)](https://github.com/opencurve/curve/releases)
+[![LICENSE](https://img.shields.io/badge/licence-Apache--2.0%2FGPL-blue)](https://github.com/opencurve/curve/blob/master/LICENSE)
 
-**A sandbox project hosted by the CNCF Foundation**
+Curve is a distributed storage system designed and developed by NetEase, featured with high performance, easy operation and cloud native. Curve is compose of CurveBS(Curve Block Storage) and CurveFS(Curve FileSystem). CurveBS supports snapshot, clone, and recover, also supports virtual machines with qemu and physical machine with nbd. CurveFS supports POSIX based on Fuse.
 
-#### English | [简体中文](README_cn.md)
-### 📄 [Documents](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7)
-<div align=left>
+## Curve Block Service vs Ceph Block Device
 
-<div class="column" align="middle">
-  </a>
-  <a href="https://github.com/opencurve/curve/blob/master/LICENSE">
-    <img src=https://img.shields.io/aur/license/android-studio?style=plastic alt="license"/>
-  </a>
-  <a href="https://github.com/opencurve/curve/releases">
-   <img src="https://img.shields.io/github/v/release/opencurve/curve?include_prereleases" alt="release"/>
-  </a>
-  <a href="https://bestpractices.coreinfrastructure.org/projects/6136">
-    <img src="https://bestpractices.coreinfrastructure.org/projects/6136/badge">
-  </a>
-  <a href="https://github.com/opencurve/curve/tree/master/docs">
-    <img src="https://img.shields.io/badge/docs-latest-green.svg">
-</div>
+Curve: v1.2.0
 
-✨ Contents
-========
+Ceph: L/N
+### Performance
+Curve random read and write performance far exceeds Ceph in the block storage scenario.
 
-  - [About Curve](#about-curve)
-  - [Curve Architecture](#curve-architecture)
-  - [Design Documentation](#design-documentation)
-  - [CurveBS quick start](#curvebs-quick-start)
-    - [Deploy an All-in-one experience environment](#deploy-an-all-in-one-experience-environment)
-    - [FIO curve block storage engine](#fio-curve-block-storage-engine)
-  - [CurveFS quick start](#curvefs-quick-start)
-  - [Test environment configuration](#test-environment-configuration)
-  - [Governance](#governance)
-  - [Contribute us](#contribute-us)
-  - [Code of Conduct](#code-of-conduct)
-  - [LICENSE](#license)
-  - [Release Cycle](#release-cycle)
-  - [Branch](#branch)
-  - [Contributors](#contributors)
-  - [Star History](#star-history)
-  - [Feedback & Contact](#feedback--contact)
+Environment：3 replicas on a 6-node cluster, each node has 20xSATA SSD, 2xE5-2660 v4 and 256GB memory.
 
-##  About Curve
+Single Vol：
+<image src="docs/images/1-nbd-en.png">
 
-**Curve** is a modern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
+Multi Vols：
+<image src="docs/images/10-nbd-en.png">
 
-The core application scenarios of CurveBS mainly include:
-- the performance, mixed, capacity cloud disk or persistent volume of virtual machine/container, and remote disks of physical machines
-- high-performance separation of storage and computation architecture: high-performance and low latency architecture based on RDMA+SPDK, supporting the separation deployment structure of various databases such as MySQL and Kafka
+### Stability
+The stability of the common abnormal Curve is better than that of Ceph in the block storage scenario.
+| Fault Case | One Disk Failure | Slow Disk Detect | One Server Failure | Server Suspend Animation |
+| :----: | :----: | :----: | :----: | :----: |
+| Ceph | jitter 7s | Continuous io jitter | jitter 7s | unrecoverable |
+| Curve | jitter 4s | no effect | jitter 4s | jitter 4s |
+### Ops
+Curve ops is more friendly than Curve in the block storage scenario.
+| Ops scenarios | Upgrade clients | Balance |
+| :----: | :----: | :----: |
+| Ceph | do not support live upgrade | via plug-in with IO influence |
+| Curve | support live upgrade with second jitter | auto with no influence on IO |
 
-
-The core application scenarios of CurveFS mainly include:
-- the cost-effective storage in AI training scene
-- the hot and cold data automation layered storage in big data scenarios
-- the cost-effective shared file storage on the public cloud: It can be used for business scenarios such as AI, big data, file sharing
-- Hybrid storage: Hot data is stored in the local IDC, cold data is stored in public cloud
-
-<details>
-  <summary><b><font=5>High Performance | More stable | Easy Operation | Cloud Native</b></font></summary>
-
-- High Performance : CurveBS vs CephBS
-
-  CurveBS: v1.2.0
-
-  CephBS: L/N
-  Performance:
-  CurveBS random read and write performance far exceeds CephBS in the block storage scenario.
-
-  Environment：3 replicas on a 6-node cluster, each node has 20xSATA SSD, 2xE5-2660 v4 and 256GB memory.
-
-  Single Vol：
-  <image src="docs/images/1-nbd-en.png">
-
-  Multi Vols：
-  <image src="docs/images/10-nbd-en.png">
-
-- More stable
-  - The stability of the common abnormal Curve is better than that of Ceph in the block storage scenario.
-    | Fault Case | One Disk Failure | Slow Disk Detect | One Server Failure | Server Suspend Animation |
-    | :----: | :----: | :----: | :----: | :----: |
-    | CephBS | jitter 7s | Continuous io jitter | jitter 7s | unrecoverable |
-    | CurveBS | jitter 4s | no effect | jitter 4s | jitter 4s |
-
-- Easy Operation
-  - We have developed [CurveAdm](https://github.com/opencurve/curveadm/wiki) to help O&M staff.
-    | tools  |CephAdm | CurveAdm|
-    | :--: | :--: |:--: |
-    | easy Installation |  ✔️ |   ✔️ |
-    | easy Deployment| ❌(slightly more steps) |  ✔️ |
-    | playground | ❌|   ✔️|
-    | Multi-Cluster Management | ❌ |   ✔️ |
-    | easy Expansion | ❌(slightly more steps)|   ✔️|
-    |easy Upgrade |  ✔️ |   ✔️|
-    |easy to stop service | ❌ |   ✔️|
-    |easy Cleaning | ❌ |   ✔️ |
-    |Deployment environment testing|  ❌ |  ✔️ |
-    |Operational audit|  ❌ |   ✔️|
-    |Peripheral component deployment| ❌ |   ✔️|
-    |easy log reporting| ❌ |   ✔️|
-    |Cluster status statistics reporting| ❌|   ✔️|
-    |Error code classification and solutions|  ❌ |   ✔️|
-  - Ops
-    CurveBS ops is more friendly than CephBS in the block storage scenario.
-    | Ops scenarios | Upgrade clients | Balance |
-    | :----: | :----: | :----: |
-    | CephBS | do not support live upgrade | via plug-in with IO influence |
-    | CurveBS | support live upgrade with second jitter | auto with no influence on IO |
-
-- Cloud Native
-  - Please see [Our understanding of cloud native](https://github.com/opencurve/curve/wiki/Roadmap).
-
-  </details>
-<details>
-  <summary><b><font=5>Docking OpenStack</b></font></summary>
-
-- Please see [Curve-cinder](https://github.com/opencurve/curve-cinder).
-
-
-</details>
-<details>
-  <summary><b><font=5>Docking Kubernetes</b></font></summary>
-
-- Use [Curve CSI Driver](https://github.com/opencurve/curve-csi), The plugin implements the Container Storage Interface(CSI) between Container Orchestrator(CO) and Curve cluster. It allows dynamically provisioning curve volumes and attaching them to workloads.
-- For details of the documentation, see [CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md).
-</details>
-<details>
-  <summary><b><font=5>Docking PolarDB | PG </b></font></summary>
-
-- It serves as the underlying storage base for [polardb for postgresql](https://github.com/ApsaraDB/PolarDB-for-PostgreSQL) in the form of storage and computation separation, providing data consistency assurance for upper layer database applications, extreme elasticity scaling, and high performance HTAP.
-
-- Deployment details can be found at [PolarDB | PG Advanced Deployment(CurveBS)](https://apsaradb.github.io/PolarDB-for-PostgreSQL/zh/deploying/storage-curvebs.html).
-
-
-</details>
-<details>
-  <summary><b><font=5> More...</b></font></summary>
-
-- Curve can also be used as cloud storage middleware using S3-compatible object storage as the data storage engine, providing cost-effective shared file storage for public cloud users.
-</details>
-
-## Curve Architecture
-
-<div align=center> <image src="docs/images/Curve-arch.png" width=60%>
-<div align=left>
-
-<details>
-  <summary><b><font=4>Curve on Hybrid Cloud</b></font></summary>
-
-Curve supports deployment in private and public cloud environments, and can also be used in a hybrid cloud:
-<div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
-<div align=left>
-
-One of them, CurveFS shared file storage system, can be elastically scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
-
-</details>
-
-<div align=left>
-
-<details>
-  <summary><b><font=4>Curve on Public Cloud</b></font></summary>
-
-In a public cloud environment, users can deploy CurveFS clusters to replace the shared file storage system provided by cloud vendors and use cloud disks for acceleration, which can greatly reduce business costs, with the following deployment architecture:
-<div align=center>
- <image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
-
-</details>
-<div align=left>
-
-##  Design Documentation
+## Design Documentation
 
 - Wanna have a glance at Curve? Click here for [Intro to Curve](https://www.opencurve.io/)!
 - Want more details about CurveBS? Our documentation for every component:
@@ -188,84 +59,75 @@ In a public cloud environment, users can deploy CurveFS clusters to replace the
   - [Client Python API](docs/en/curve-client-python-api_en.md)
 - Application based on CurveBS
   - [Work with k8s](docs/en/k8s_csi_interface_en.md)
-- CurveFS documentations
-  - [Architecture design](docs/cn/curvefs_architecture.md)
-  - [Client design](docs/cn/curvefs-client-design.md)
-  - [Metadata management](docs/cn/curvefs-metaserver-overview.md)
+- Want more details about CurveFS? Our documentation for every component:
+  - [Architecture design](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/CurveFS%E6%96%B9%E6%A1%88%E8%AE%BE%E8%AE%A1%EF%BC%88%E6%80%BB%E4%BD%93%E8%AE%BE%E8%AE%A1%EF%BC%8C%E5%8F%AA%E5%AE%9E%E7%8E%B0%E4%BA%86%E9%83%A8%E5%88%86%EF%BC%89.pdf)
+  - [Client design](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/CurveFS%20Client%20%E6%A6%82%E8%A6%81%E8%AE%BE%E8%AE%A1.pdf)
+  - [Metadata management](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E5%85%83%E6%95%B0%E6%8D%AE%E7%AE%A1%E7%90%86.pdf)
   - [Data caching](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%94%AF%E6%8C%81S3%20%E6%95%B0%E6%8D%AE%E7%BC%93%E5%AD%98%E6%96%B9%E6%A1%88.pdf)
   - [Space allocation](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E7%A9%BA%E9%97%B4%E5%88%86%E9%85%8D%E6%96%B9%E6%A1%88.pdf)
   - [more details](https://github.com/opencurve/curve-meetup-slides/tree/main/CurveFS)
 
-- CurveAdm
-  - [CurveAdm Doc](https://github.com/opencurve/curveadm/wiki)
+## Quick Start of CurveBS
 
-##  CurveBS quick start
-In order to improve the operation and maintenance convenience of Curve, we designed and developed the [CurveAdm](https://github.com/opencurve/curveadm) project, which is mainly used for deploying and managing Curve clusters. Currently, it supports the deployment of CurveBS & CurveFS (scaleout, upgrade and other functions are under development), please refer to the [CurveAdm User Manual](https://github.com/opencurve/curveadm/wiki) for related documentation, and install the CurveAdm tool according to the manual before deploying the Curve cluster.
+Want to try on it? Take it easy! We'll help you step by step, but make sure you've read this [Tips](docs/en/deploy_en.md#Tips) before you start.
 
+### Deploy an all-in-one environment (to try how CURVE works)
 
-###  Deploy an All-in-one experience environment
+[Deploy on single machine](docs/en/deploy_en.md#deploy-on-single-machine)
 
-Please refer to the [CurveBS cluster deployment steps](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment) in the CurveAdm user manual. For standalone experience, please use the "Cluster Topology File - Standalone Deployment" template.
+### Deploy multi-machine cluster (try it in production environment)
 
-[The command tools' instructions](docs/cn/curve_ops_tool.md)
-### FIO Curve block storage engine
-Fio Curve engine is added, you can clone https://github.com/opencurve/fio and compile the fio tool with our engine(depend on nebd lib), fio command line example:
-```bash
-$ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
-```
+[Deploy on multiple machines](docs/en/deploy_en.md#deploy-on-multiple-machines)
 
-If you have any questions during performance testing, please check the [Curve block storage performance tuning guide](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md).
+### curve_ops_tool introduction
 
-##  CurveFS quick start
-Please use [CurveAdm](https://github.com/opencurve/curveadm/wiki) tool to deploy CurveFS，see [CurveFS Deployment Process](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), and the [CurveFS Command Instructions](curvefs/src/tools#readme).
+[curve_ops_tool introduction](docs/en/curve_ops_tool_en.md)
 
-## Test environment configuration
+## Quick Start of CurveFS
+In order to improve the convenience of Curve operation and maintenance, we have designed and developed the [CurveAdm](https://github.com/opencurve/curveadm) project, which is mainly used to deploy and manage Curve clusters. Currently, it supports the deployment of CurveFS ​​(CurveBS support is under development).
 
-Please refer to the [Test environment configuration](docs/cn/测试环境配置信息.md)
+Detail for CurveFS deploy: [CurveFS ​​deployment](https://github.com/opencurve/curveadm#deploy-cluster)
 
-## Practical
-- [CurveBS+NFS Build NFS Server](docs/practical/curvebs_nfs.md)
-- [CurveFS+MinIO S3 Gateway](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
+## For Developers
 
-## Governance
-See [Governance](https://github.com/opencurve/community/blob/master/GOVERNANCE.md).
+### Deploy build and development environment
 
-##  Contribute us
-Participation in the Curve project is described in the [Curve Developers Guidelines](developers_guide.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
-We welcome your contribution!
+[development environment deployment](docs/en/build_and_run_en.md)
 
-## Code of Conduct
-Curve follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
+### Compile test cases and run
+[test cases compiling and running](docs/en/build_and_run_en.md#test-case-compilation-and-execution)
 
-## LICENSE
-Curve is distributed under the [**Apache 2.0 LICENSE**](LICENSE).
+### FIO curve block storage engine
+Fio curve engine is added, you can clone https://github.com/skypexu/fio/tree/nebd_engine and compile the fio tool with our engine(depend on nebd lib), fio command line example: `./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test`
 
-##  Release Cycle
+### Coding style guides
+CURVE is coded following [Google C++ Style Guide strictly](https://google.github.io/styleguide/cppguide.html). Please follow this guideline if you're trying to contribute your codes.
+
+### Code coverage requirement
+1. Unit tests: Incremental line coverage ≥ 80%, incremental branch coverage ≥ 70%
+2. Integration tests: Measure together with unit tests, and should fulfill the same requirement
+3. Exception tests: Not required yet
+
+### Other processes
+
+After finishing the development of your code, you should submit a pull request to master branch of CURVE and fill out a pull request template. The pull request will trigger the CI automatically, and the code will only be merged after passing the CI and being reviewed.
+
+For more detail, please refer to [CONTRIBUTING](https://github.com/opencurve/curve/blob/master/CONTRIBUTING.md).
+
+## Release Cycle
 - CURVE release cycle：Half a year for major version, 1~2 months for minor version
 
 - Versioning format: We use a sequence of three digits and a suffix (x.y.z{-suffix}), x is the major version, y is the minor version, and z is for bugfix. The suffix is for distinguishing beta (-beta), RC (-rc) and GA version (without any suffix). Major version x will increase 1 every half year, and y will increase every 1~2 months. After a version is released, number z will increase if there's any bugfix.
 
 ## Branch
+
 All the developments will be done under master branch. If there's any new version to establish, a new branch release-x.y will be pulled from the master, and the new version will be released from this branch.
 
-## Contributors
-This project exists thanks to all the people who contribute.
-
-<a href="https://github.com/opencurve/curve/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=opencurve/curve" />
-</a>
-
-Made with [contrib.rocks](https://contrib.rocks).
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=opencurve/curve&type=Date)](https://star-history.com/#opencurve/curve&Date)
-
 ## Feedback & Contact
+
 - [Github Issues](https://github.com/openCURVE/CURVE/issues)：You are sincerely welcomed to issue any bugs you came across or any suggestions through Github issues. If you have any question you can refer to our FAQ or join our user group for more details.
 - [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：Frequently asked question in our user group, and we'll keep working on it.
 - User group：We use Wechat group currently.
 - [Double Week Meetings](https://github.com/opencurve/curve-meetup-slides/tree/main/2022): We have an online community meeting every two weeks which talk about what Curve is doing and planning to do. The time and links of the meeting are public in the user group and [Double Week Meetings](https://github.com/opencurve/curve-meetup-slides/tree/main/2022).
 
-
-<img src="docs/images/curve-wechat.jpeg" style="zoom: 65%;" />
+<img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />

README_cn.md

@@ -1,181 +1,52 @@
+[English version](README.md)
 
 
-<div align=center> <img src="docs/images/curve-logo1-nobg.png" width = 45%>
+<img src="docs/images/curve-logo1.png"/>
 
-<div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
+# Curve
 
-**云原生高性能分布式存储系统**
-
-**CNCF基金会的沙箱托管项目**
-
-#### [English](README.md) | 简体中文
-### 📄 [文档](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7)
-<div align=left>
-
-<div class="column" align="middle">
-  </a>
-  <a href="https://github.com/opencurve/curve/blob/master/LICENSE">
-    <img src=https://img.shields.io/aur/license/android-studio?style=plastic alt="license"/>
-  </a>
-  <a href="https://github.com/opencurve/curve/releases">
-   <img src="https://img.shields.io/github/v/release/opencurve/curve?include_prereleases" alt="release"/>
-  </a>
-  <a href="https://bestpractices.coreinfrastructure.org/projects/6136">
-    <img src="https://bestpractices.coreinfrastructure.org/projects/6136/badge">
-  </a>
-  <a href="https://github.com/opencurve/curve/tree/master/docs">
-    <img src="https://img.shields.io/badge/docs-latest-green.svg">
-</div>
-
-✨ 目录
-========
-
-  - [关于 Curve](#关于-curve)
-  - [Curve 架构](#curve-架构)
-  - [设计文档](#设计文档)
-  - [CurveBS 快速体验](#curvebs-快速体验)
-    - [部署All-in-one体验环境](#部署all-in-one体验环境)
-    - [FIO curve块存储引擎](#fio-curve块存储引擎)
-  - [CurveFS 快速体验](#curvefs-快速体验)
-  - [测试环境配置](#测试环境配置)
-  - [社区治理](#社区治理)
-  - [贡献我们](#贡献我们)
-  - [行为守则](#行为守则)
-  - [LICENSE](#license)
-  - [版本发布周期](#版本发布周期)
-  - [分支规则](#分支规则)
-  - [反馈及交流](#反馈及交流)
+[![Jenkins Coverage](https://img.shields.io/jenkins/coverage/cobertura?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_untest_job%2F)](http://59.111.91.248:8080/job/curve_untest_job/HTML_20Report/)
+[![Robot failover](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_failover_testjob%2F&label=failover)](http://59.111.91.248:8080/job/curve_failover_testjob/)
+[![Robot interface](https://img.shields.io/jenkins/tests?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_robot_job%2F)](http://59.111.91.248:8080/job/curve_robot_job/)
+[![BUILD Status](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fopencurve_multijob%2F)](http://59.111.91.248:8080/job/opencurve_multijob/lastBuild)
+[![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://github.com/opencurve/curve/tree/master/docs)
+[![Releases](https://img.shields.io/github/v/release/opencurve/curve?include_prereleases)](https://github.com/opencurve/curve/releases)
+[![LICENSE](https://img.shields.io/badge/licence-Apache--2.0%2FGPL-blue)](https://github.com/opencurve/curve/blob/master/LICENSE)
 
 
-##  关于 Curve
-**Curve** 是网易主导自研的现代化存储系统, 目前支持文件存储(CurveFS)和块存储(CurveBS)。现作为沙箱项目托管于[CNCF](https://www.cncf.io/)。
+Curve是网易自主设计研发的高性能、易运维、云原生的分布式存储系统，目前提供块(CurveBS)和文件(CurveFS)两种存储方式。CurveBS支持快照克隆和恢复,支持QEMU虚拟机和物理机NBD设备两种挂载方式。CurveFS基于Fuse支持POSIX文件系统接口。
 
-CurveBS的核心应用场景主要包括：
-- 虚拟机/容器的性能型、混合型、容量型云盘或持久化卷，以及物理机的远程存储盘
-- 高性能存算分离架构：基于RDMA+SPDK的高性能低时延架构，支撑MySQL、kafka等各类数据库、中间件的存算分离部署架构，提升实例交付效率和资源利用率
+## Curve Block Service vs Ceph Block Device
 
+Curve: v1.2.0
 
-CurveFS的核心应用场景主要包括：
-- AI训练（含机器学习等）场景下的高性价比存储
-- 大数据场景下的冷热数据自动化分层存储
-- 公有云上高性价比的共享文件存储：可用于AI、大数据、文件共享等业务场景
-- 混合云存储：热数据存储在本地IDC，冷数据存储在公有云
+Ceph: L/N
+### 性能
+块存储场景下，Curve随机读写性能远优于Ceph。
+测试环境：6台服务器*20块SATA SSD，E5-2660 v4，256G，3副本，使用nbd场景。
 
+单卷场景：
+<image src="docs/images/1-nbd.jpg">
 
-<details>
-  <summary><b><font=5>高性能 | 更稳定 | 易运维 | 云原生</b></font></summary>
-
-- 高性能 : CurveBS vs CephBS
-
-  CurveBS: v1.2.0
-
-  CephBS: L/N
-  性能:
-  块存储场景下，CurveBS随机读写性能远优于CephBS。
-  测试环境：6台服务器*20块SATA SSD，E5-2660 v4，256G，3副本，使用nbd场景。
-
-  单卷场景：
-  <image src="docs/images/1-nbd.jpg">
-
-  多卷场景：
-  <image src="docs/images/10-nbd.jpg">
-
-- 更稳定
-  - 块存储场景下，常见异常CurveBS的稳定性优于CephBS。
-
-    | 异常场景 | 单盘故障 | 慢盘 | 机器宕机 | 机器卡住 |
-    | :----: | :----: | :----: | :----: | :----: |
-    | CephBS | 抖动7s | 持续io抖动 | 抖动7s | 不可恢复 |
-    | CurveBS | 抖动4s | 无影响 | 抖动4s | 抖动4s |
-
-- 易运维
-  - 我们开发了 [CurveAdm](https://github.com/opencurve/curveadm/wiki)来帮助运维人员。
-
-    | 工具 |CephAdm | CurveAdm|
-    | :--:  | :--: |:--: |
-    | 一键安装  |   ✔️ |   ✔️ |
-    | 一键部署  | ❌(步骤稍多) |  ✔️ |
-    | playground |  ❌|   ✔️|
-    | 多集群管理  |  ❌ |   ✔️ |
-    | 一键扩容 | ❌(步骤稍多)|   ✔️|
-    |一键升级 |   ✔️ |   ✔️|
-    |一键停服 |  ❌ |   ✔️|
-    |一键清理 |  ❌ |   ✔️ |
-    |部署环境检测|  ❌ |   ✔️ |
-    |操作审计| ❌ |   ✔️|
-    |周边组件部署| ❌ |   ✔️|
-    |一键日志上报| ❌ |   ✔️|
-    |集群状态统计上报|  ❌|   ✔️|
-    |错误码分类及解决方案|  ❌ |   ✔️|
-  - 运维
-    块存储场景下，CurveBS常见运维更友好。
-
-    | 运维场景 | 客户端升级 | 均衡 |
-    | :----: | :----: | :----: |
-    | CephBS | 不支持热升级 | 外部插件调整，影响业务IO |
-    | CurveBS | 支持热升级，秒级抖动 | 自动均衡，对业务IO无影响 |
-- 云原生
-  - 详见[我们对云原生的理解](https://github.com/opencurve/curve/wiki/Roadmap_CN)。
-
-  </details>
-<details>
-  <summary><b><font=5>对接 OpenStack</b></font></summary>
-
-- 详见 [Curve-cinder](https://github.com/opencurve/curve-cinder)。
-
-
-</details>
-<details>
-  <summary><b><font=5>对接 Kubernetes</b></font></summary>
-
-- 使用 [Curve CSI Driver](https://github.com/opencurve/curve-csi) 插件在 Container Orchestrator (CO) 与 Curve 集群中实现了 Container Storage Interface(CSI)。
-- 文档详见[CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md)。
-</details>
-<details>
-  <summary><b><font=5>对接 PolarDB | PG </b></font></summary>
-
-- 作为存算分离形态分布式数据库 [PolarDB | PG](https://github.com/ApsaraDB/PolarDB-for-PostgreSQL) 底层存储底座, 为上层数据库应用提供数据一致性保证, 极致弹性, 高性能HTAP。部署详见[PolarDB | PG 进阶部署(CurveBS)](https://apsaradb.github.io/PolarDB-for-PostgreSQL/zh/deploying/storage-curvebs.html)。
-
-
-</details>
-<details>
-  <summary><b><font=5> 更多...</b></font></summary>
-
-- Curve 亦可作为云存储中间件使用 S3 兼容的对象存储作为数据存储引擎，为公有云用户提供高性价比的共享文件存储。
-</details>
-
-## Curve 架构
-
-<div align=center> <image src="docs/images/Curve-arch.png" width=60%>
-<div align=left>
-
-<details>
-  <summary><b><font=4>Curve混合云支持</b></font></summary>
-
-Curve支持部署在私有云和公有云环境，也可以以混合云方式使用，私有云环境下的部署架构如下：
-<div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
-<div align=left>
-
-其中CurveFS共享文件存储系统可以弹性伸缩到公有云存储，可以为用户提供更大的容量弹性、更低的成本、更好的性能体验。
-
-</details>
-
-<div align=left>
-
-<details>
-  <summary><b><font=4>Curve公有云支持</b></font></summary>
-
-公有云环境下，用户可以部署CurveFS集群，用来替换云厂商提供的共享文件存储系统，并利用云盘进行加速，可极大的降低业务成本，其部署架构如下：
-<div align=center>
- <image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
-
-</details>
-<div align=left>
+多卷场景：
+<image src="docs/images/10-nbd.jpg">
 
+### 稳定性
+块存储场景下，常见异常Curve的稳定性优于Ceph。
+| 异常场景 | 单盘故障 | 慢盘 | 机器宕机 | 机器卡住 |
+| :----: | :----: | :----: | :----: | :----: |
+| Ceph | 抖动7s | 持续io抖动 | 抖动7s | 不可恢复 |
+| Curve | 抖动4s | 无影响 | 抖动4s | 抖动4s |
+### 运维
+块存储场景下，Curve常见运维更友好。
+| 运维场景 | 客户端升级 | 均衡 |
+| :----: | :----: | :----: |
+| Ceph | 不支持热升级 | 外部插件调整，影响业务IO |
+| Curve | 支持热升级，秒级抖动 | 自动均衡，对业务IO无影响 |
 
 ## 设计文档
 
-- 通过 [Curve概述](https://opencurve.github.io/) 可以了解 Curve 架构。
+- 通过 [Curve概述](https://opencurve.github.io/) 可以了解 Curve 架构
 - CurveBS相关文档
   - [NEBD](docs/cn/nebd.md)
   - [MDS](docs/cn/mds.md)
@@ -188,64 +59,69 @@ Curve支持部署在私有云和公有云环境，也可以以混合云方式使
 - CurveBS上层应用
   - [对接k8s文档](docs/cn/k8s_csi_interface.md)
 - CurveFS相关文档
-  - [架构设计](docs/cn/curvefs_architecture.md)
-  - [Client概要设计](docs/cn/curvefs-client-design.md)
-  - [元数据管理](docs/cn/curvefs-metaserver-overview.md)
+  - [架构设计](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/CurveFS%E6%96%B9%E6%A1%88%E8%AE%BE%E8%AE%A1%EF%BC%88%E6%80%BB%E4%BD%93%E8%AE%BE%E8%AE%A1%EF%BC%8C%E5%8F%AA%E5%AE%9E%E7%8E%B0%E4%BA%86%E9%83%A8%E5%88%86%EF%BC%89.pdf)
+  - [Client概要设计](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/CurveFS%20Client%20%E6%A6%82%E8%A6%81%E8%AE%BE%E8%AE%A1.pdf)
+  - [元数据管理](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E5%85%83%E6%95%B0%E6%8D%AE%E7%AE%A1%E7%90%86.pdf)
   - [数据缓存方案](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%94%AF%E6%8C%81S3%20%E6%95%B0%E6%8D%AE%E7%BC%93%E5%AD%98%E6%96%B9%E6%A1%88.pdf)
   - [空间分配方案](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E7%A9%BA%E9%97%B4%E5%88%86%E9%85%8D%E6%96%B9%E6%A1%88.pdf)
   - [更多](https://github.com/opencurve/curve-meetup-slides/tree/main/CurveFS)
-- CurveAdm相关文档
-  - [Wiki](https://github.com/opencurve/curveadm/wiki)
 
-##  CurveBS 快速体验
-为了提升 Curve 的运维便利性，我们设计开发了 [CurveAdm](https://github.com/opencurve/curveadm) 项目，其主要用于部署和管理 Curve 集群，目前已支持部署CurveBS & CurveFS（扩容、版本升级等更多功能正在开发中），相关使用文档请参考 [CurveAdm用户手册](https://github.com/opencurve/curveadm/wiki)，并根据手册首先安装CurveAdm工具之后再进行Curve集群的部署。
+## CurveBS快速开始
 
-###  部署All-in-one体验环境
-请参考CurveAdm用户手册中[CurveBS集群部署步骤](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment)，单机体验环境请使用“集群拓扑文件-单机部署”模板。
+在您开始动手部署前请先仔细阅读特别说明部分：[特别说明](docs/cn/deploy.md#%E7%89%B9%E5%88%AB%E8%AF%B4%E6%98%8E)
 
-curve 提供了命令行工具以查看集群状态和进行基本集群操作:[命令行工具说明](docs/cn/curve_ops_tool.md)
-### FIO Curve块存储引擎
-fio的Curve块存储引擎代码已经上传到 https://github.com/opencurve/fio ，请自行编译测试（依赖nebd库），fio命令行示例：
-```bash
-$ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
-```
+### 部署All-in-one体验环境
 
-在性能测试过程中有任何问题，请查看[Curve块存储性能调优指南](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md)
+[单机部署](docs/cn/deploy.md#%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2)
 
-##  CurveFS 快速体验
-请使用 [CurveAdm](https://github.com/opencurve/curveadm/wiki) 工具进行 CurveFS 的部署，具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), 以及[CurveFS命令行工具说明](curvefs/src/tools#readme)。
+### 部署多机集群
 
-## 测试环境配置
+[多机部署](docs/cn/deploy.md#%E5%A4%9A%E6%9C%BA%E9%83%A8%E7%BD%B2)
 
-请参考 [测试环境配置](docs/cn/测试环境配置信息.md)
-##  社区治理
-请参考[社区治理](https://github.com/opencurve/community/blob/master/GOVERNANCE.md)。
+### 查询工具说明
 
-##  贡献我们
+[查询工具说明](docs/cn/curve_ops_tool.md)
 
-参与 Curve 项目开发详见[Curve 开发者指南](developers_guide_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
+## CurveFS快速开始
+为了提升 Curve 的运维便利性，我们设计开发了 [CurveAdm](https://github.com/opencurve/curveadm) 项目，其主要用于部署和管理 Curve 集群，目前已支持部署 CurveFS（CurveBS 的支持正在开发中）。
 
-## 最佳实践
-- [CurveBS+NFS搭建NFS存储](docs/practical/curvebs_nfs.md)
-- [CurveFS+S3网关部署实践](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
+具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm#deploy-cluster)
 
-##  行为守则
-Curve 的行为守则遵循[CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。
+## 参与开发
 
-## LICENSE
-Curve 在 [Apache 2.0](LICENSE) 协议下进行分发。
+### 部署编译开发环境
 
-##  版本发布周期
+[编译开发环境搭建](docs/cn/build_and_run.md)
+
+### 测试用例编译及运行
+[测试用例编译及运行](docs/cn/build_and_run.md#%E6%B5%8B%E8%AF%95%E7%94%A8%E4%BE%8B%E7%BC%96%E8%AF%91%E5%8F%8A%E6%89%A7%E8%A1%8C)
+
+### FIO curve块存储引擎
+fio的curve块存储引擎代码已经上传到 https://github.com/skypexu/fio/tree/nebd_engine ，请自行编译测试（依赖nebd库），fio命令行示例：`./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test`
+
+### 编码规范
+CURVE编码规范严格按照[Google C++开源项目编码指南](https://zh-google-styleguide.readthedocs.io/en/latest/google-cpp-styleguide/contents/)来进行代码编写，请您也遵循这一指南来提交您的代码。
+
+### 测试覆盖率要求
+1. 单元测试：增量行覆盖80%以上，增量分支覆盖70%以上
+2. 集成测试：与单元测试合并统计，满足上述覆盖率要求即可
+3. 异常测试：暂不做要求
+
+### 其他开发流程说明
+代码开发完成之后，提[pr](https://github.com/opencurve/curve/compare)到curve的master分支。提交pr时，请填写pr模板。pr提交之后会自动触发CI，CI通过并且经过review之后，代码才可合入。
+具体规则请见[CONTRIBUTING](https://github.com/opencurve/curve/blob/master/CONTRIBUTING.md).
+
+## 版本发布周期
 - CURVE版本发布周期：大版本半年，小版本1~2个月
 - 版本号规则：采用3段式版本号，x.y.z{-后缀}，x是大版本，y是小版本，z是bugfix，后缀用来区beta版本(-beta)、rc版本(-rc)、和稳定版本(没有后缀)。每半年的大版本是指x增加1，每1~2个月的小版本是y增加1。正式版本发布之后，如果有bugfix是z增加1。
 
-##  分支规则
+## 分支规则
 所有的开发都在master分支开发，如果需要发布版本，从master拉取新的分支**release-x.y**。版本发布从release-x.y分支发布。
 
-##  反馈及交流
+## 反馈及交流
 
-- [Github Issues](https://github.com/openCURVE/CURVE/issues)：欢迎提交BUG、建议，使用中如遇到问题可参考FAQ或加入我们的User group进行咨询。
-- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：主要根据User group中常见问题整理，还在逐步完善中。
+- [Github Issues](https://github.com/openCURVE/CURVE/issues)：欢迎提交BUG、建议，使用中如遇到问题可参考FAQ或加入我们的User group进行咨询
+- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：主要根据User group中常见问题整理，还在逐步完善中
 - User group：当前为微信群，由于群人数过多，需要先添加以下个人微信，再邀请进群。
 
 <img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />
@@ -254,3 +130,4 @@ Curve 在 [Apache 2.0](LICENSE) 协议下进行分发。
 
 
 
+

WORKSPACE

@@ -16,25 +16,12 @@
 
 workspace(name = "curve")
 
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-# skylib
-http_archive(
-    name = "bazel_skylib",
-    urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
-    ],
-    sha256 = "af87959afe497dc8dfd4c6cb66e1279cb98ccc84284619ebfec27d9c09a903de",
-)
-load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
-bazel_skylib_workspace()
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 git_repository(
     name = "com_github_baidu_braft",
     remote = "https://github.com/baidu/braft",
-    commit = "d12de388c97998f5ccd5cb97ed0da728815ef438",
+    commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e",
 )
 
 bind(
@@ -45,28 +32,11 @@ bind(
 # proto_library, cc_proto_library, and java_proto_library rules implicitly
 # depend on @com_google_protobuf for protoc and proto runtimes.
 # This statement defines the @com_google_protobuf repo.
-
-# zlib
-http_archive(
-    name = "net_zlib",
-    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
-    strip_prefix = "zlib-1.2.11",
-    urls = ["https://zlib.net/zlib-1.2.11.tar.gz"],
-)
-
-bind(
-    name = "zlib",
-    actual = "@net_zlib//:zlib",
-)
-
 http_archive(
     name = "com_google_protobuf",
-    strip_prefix = "protobuf-3.6.1.3",
-    patch_args = ["-p1"],
-    patches = ["//:thirdparties/protobuf/protobuf.patch"],
-    sha256 = "9510dd2afc29e7245e9e884336f848c8a6600a14ae726adb6befdb4f786f0be2",
-    urls = ["https://github.com/google/protobuf/archive/v3.6.1.3.zip"],
+    sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30",
+    strip_prefix = "protobuf-3.5.0",
+    urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"],
 )
 
 bind(
@@ -75,14 +45,11 @@ bind(
 )
 
 #import the  gtest files.
-http_archive(
+new_git_repository(
     name = "com_google_googletest",
-    urls = [
-        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
-        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
-    ],
-    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
-    strip_prefix = "googletest-release-1.12.1",
+    build_file = "bazel/gmock.BUILD",
+    remote = "https://github.com/google/googletest",
+    tag = "release-1.8.0",
 )
 
 bind(
@@ -109,7 +76,10 @@ bind(
 http_archive(
     name = "com_github_gflags_gflags",
     strip_prefix = "gflags-2.2.2",
-    urls = ["https://github.com/gflags/gflags/archive/v2.2.2.tar.gz"],
+    urls = [
+        "https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz",
+        "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz",
+    ],
 )
 
 bind(
@@ -117,11 +87,11 @@ bind(
     actual = "@com_github_gflags_gflags//:gflags",
 )
 
-http_archive(
+new_http_archive(
     name = "com_github_google_leveldb",
-    build_file = "@com_github_brpc_brpc//:leveldb.BUILD",
+    build_file = "bazel/leveldb.BUILD",
     strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
-    urls = ["https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz"],
+    url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz",
 )
 
 bind(
@@ -130,7 +100,7 @@ bind(
 )
 
 git_repository(
-    name = "com_github_brpc_brpc",
+    name = "com_github_apache_brpc",
     remote = "https://github.com/apache/incubator-brpc",
     commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
     patches = ["//:thirdparties/brpc/brpc.patch"],
@@ -139,28 +109,28 @@ git_repository(
 
 bind(
     name = "brpc",
-    actual = "@com_github_brpc_brpc//:brpc",
+    actual = "@com_github_apache_brpc//:brpc",
 )
 
 bind(
     name = "butil",
-    actual = "@com_github_brpc_brpc//:butil",
+    actual = "@com_github_apache_brpc//:butil",
 )
 
 bind(
     name = "bthread",
-    actual = "@com_github_brpc_brpc//:bthread",
+    actual = "@com_github_apache_brpc//:bthread",
 )
 
 bind(
     name = "bvar",
-    actual = "@com_github_brpc_brpc//:bvar",
+    actual = "@com_github_apache_brpc//:bvar",
 )
 
 # jsoncpp
 new_git_repository(
     name = "jsoncpp",
-    build_file = "//:thirdparties/jsoncpp.BUILD",
+    build_file = "bazel/jsoncpp.BUILD",
     remote = "https://github.com/open-source-parsers/jsoncpp.git",
     tag = "1.8.4",
 )
@@ -172,44 +142,49 @@ bind(
 
 new_local_repository(
     name = "etcdclient",
-    build_file = "//:thirdparties/etcdclient.BUILD",
+    build_file = "bazel/etcdclient.BUILD",
     path = "thirdparties/etcdclient",
 )
 
-new_local_repository(
-    name = "libmemcached",
-    build_file = "//:thirdparties/memcache/memcache.BUILD",
-    path = "thirdparties/memcache/libmemcached-1.1.2",
-)
-
-
-http_archive(
+new_http_archive(
     name = "aws",
-    urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
+    urls = [
+        "https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
+        "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
+    ],
     sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
     strip_prefix = "aws-sdk-cpp-1.7.340",
     build_file = "//:thirdparties/aws/aws.BUILD",
 )
 
-http_archive(
+new_http_archive(
     name = "aws_c_common",
-    urls = ["https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz"],
+    urls = [
+        "https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
+        "https://mirror.tensorflow.org/github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
+    ],
     sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
     strip_prefix = "aws-c-common-0.4.29",
     build_file = "//:thirdparties/aws/aws-c-common.BUILD",
 )
 
-http_archive(
+new_http_archive(
     name = "aws_c_event_stream",
-    urls = ["https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz"],
+    urls = [
+        "https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
+        "https://mirror.tensorflow.org/github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
+    ],
     sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
     strip_prefix = "aws-c-event-stream-0.1.4",
     build_file = "//:thirdparties/aws/aws-c-event-stream.BUILD",
 )
 
-http_archive(
+new_http_archive(
     name = "aws_checksums",
-    urls = ["https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz"],
+    urls = [
+        "https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
+        "https://mirror.tensorflow.org/github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
+    ],
     sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
     strip_prefix = "aws-checksums-0.1.5",
     build_file = "//:thirdparties/aws/aws-checksums.BUILD",
@@ -230,53 +205,3 @@ http_archive(
   strip_prefix = "abseil-cpp-20210324.2",
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
-
-# fmt
-http_archive(
-  name = "fmt",
-  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
-  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
-  strip_prefix = "fmt-9.1.0",
-  build_file = "//:thirdparties/fmt.BUILD",
-)
-
-# spdlog
-http_archive(
-  name = "spdlog",
-  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
-  strip_prefix = "spdlog-1.11.0",
-  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
-  build_file = "//:thirdparties/spdlog.BUILD",
-)
-
-# Bazel platform rules.
-http_archive(
-    name = "platforms",
-    sha256 = "b601beaf841244de5c5a50d2b2eddd34839788000fa1be4260ce6603ca0d8eb7",
-    strip_prefix = "platforms-98939346da932eef0b54cf808622f5bb0928f00b",
-    urls = ["https://github.com/bazelbuild/platforms/archive/98939346da932eef0b54cf808622f5bb0928f00b.zip"],
-)
-
-# RocksDB
-new_local_repository(
-    name = "rocksdb",
-    build_file = "//:thirdparties/rocksdb.BUILD",
-    path = "thirdparties/rocksdb",
-)
-
-# Hedron's Compile Commands Extractor for Bazel
-# https://github.com/hedronvision/bazel-compile-commands-extractor
-http_archive(
-    name = "hedron_compile_commands",
-
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
-    # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
-    urls = [
-        "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
-        "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
-    ],
-    strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
-)
-load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
-hedron_compile_commands_setup()

bazel/gflags.BUILD

@@ -1,5 +1,5 @@
 #
-#  Copyright (c) 2023 NetEase Inc.
+#  Copyright (c) 2020 NetEase Inc.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -14,21 +14,21 @@
 #  limitations under the License.
 #
 
+# Bazel (http://bazel.io/) BUILD file for gflags.
+#
+# See INSTALL.md for instructions for adding gflags to a Bazel workspace.
 
-load("//:copts.bzl", "CURVE_TEST_COPTS")
+licenses(["notice"])
 
-cc_test(
-   name = "client_metric_test",
-   srcs = glob([
-       "*.cpp",
-       "*.h"],
-   ),
-   copts = CURVE_TEST_COPTS,
-   deps = [
-        "//external:gtest",
-        "@com_google_googletest//:gtest",
-        "@com_google_googletest//:gtest_main",
-        "//curvefs/src/client/metric:client_metric",
-   ],
-   visibility = ["//visibility:public"],
+exports_files(["src/gflags_completions.sh", "COPYING.txt"])
+
+config_setting(
+    name = "x64_windows",
+    values = {"cpu": "x64_windows"},
 )
+
+load(":bazel/gflags.bzl", "gflags_sources", "gflags_library")
+
+(hdrs, srcs) = gflags_sources(namespace=["gflags", "google"])
+gflags_library(hdrs=hdrs, srcs=srcs, threads=0)
+gflags_library(hdrs=hdrs, srcs=srcs, threads=1)

bazel/glog.BUILD

@@ -0,0 +1,98 @@
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+cc_library(
+    name = "glog",
+    srcs = [
+        "config.h",
+        "src/base/commandlineflags.h",
+        "src/base/googleinit.h",
+        "src/base/mutex.h",
+        "src/demangle.cc",
+        "src/demangle.h",
+        "src/logging.cc",
+        "src/raw_logging.cc",
+        "src/signalhandler.cc",
+        "src/symbolize.cc",
+        "src/symbolize.h",
+        "src/utilities.cc",
+        "src/utilities.h",
+        "src/vlog_is_on.cc",
+    ] + glob(["src/stacktrace*.h"]),
+    hdrs = [
+        "src/glog/log_severity.h",
+        "src/glog/logging.h",
+        "src/glog/raw_logging.h",
+        "src/glog/stl_logging.h",
+        "src/glog/vlog_is_on.h",
+    ],
+    copts = [
+        "-Wno-sign-compare",
+        "-U_XOPEN_SOURCE",
+    ],
+    includes = ["./src"],
+    linkopts = ["-lpthread"] + select({
+        ":libunwind": ["-lunwind"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_gflags_gflags//:gflags",
+    ],
+)
+
+config_setting(
+    name = "libunwind",
+    values = {
+        "define": "libunwind=true",
+    },
+)
+
+genrule(
+    name = "run_configure",
+    srcs = [
+        "README",
+        "Makefile.in",
+        "config.guess",
+        "config.sub",
+        "install-sh",
+        "ltmain.sh",
+        "missing",
+        "libglog.pc.in",
+        "src/config.h.in",
+        "src/glog/logging.h.in",
+        "src/glog/raw_logging.h.in",
+        "src/glog/stl_logging.h.in",
+        "src/glog/vlog_is_on.h.in",
+    ],
+    outs = [
+        "config.h",
+        "src/glog/logging.h",
+        "src/glog/raw_logging.h",
+        "src/glog/stl_logging.h",
+        "src/glog/vlog_is_on.h",
+    ],
+    tools = [
+        "configure",
+    ],
+    cmd = "$(location :configure)" +
+          "&& cp -v src/config.h $(location config.h) " +
+          "&& cp -v src/glog/logging.h $(location src/glog/logging.h) " +
+          "&& cp -v src/glog/raw_logging.h $(location src/glog/raw_logging.h) " +
+          "&& cp -v src/glog/stl_logging.h $(location src/glog/stl_logging.h) " +
+          "&& cp -v src/glog/vlog_is_on.h $(location src/glog/vlog_is_on.h) "
+    ,
+)

bazel/gmock.BUILD

@@ -0,0 +1,44 @@
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+cc_library(
+      name = "gtest",
+      srcs = [
+            "googletest/src/gtest-all.cc",
+            "googlemock/src/gmock-all.cc",
+      ],
+      hdrs = glob([
+          "**/*.h",
+          "googletest/src/*.cc",
+          "googlemock/src/*.cc",
+      ]),
+      includes = [
+          "googlemock",
+          "googletest",
+          "googletest/include",
+          "googlemock/include",
+      ],
+      linkopts = ["-pthread"],
+      visibility = ["//visibility:public"],
+  )
+
+cc_library(
+      name = "gtest_main",
+      srcs = ["googlemock/src/gmock_main.cc"],
+      linkopts = ["-pthread"],
+      visibility = ["//visibility:public"],
+      deps = [":gtest"],
+)

bazel/gtest.BUILD

@@ -15,12 +15,16 @@
 #
 
 cc_library(
-    name = "spdlog",
+    name = "main",
+    srcs = glob(
+        ["src/*.cc"],
+        exclude = ["src/gtest-all.cc"]
+    ),
     hdrs = glob([
         "include/**/*.h",
+        "src/*.h"
     ]),
-    defines = ["SPDLOG_FMT_EXTERNAL"],
-    includes = ["include"],
+    copts = ["-Iexternal/gtest/include"],
+    linkopts = ["-pthread"],
     visibility = ["//visibility:public"],
-    deps = ["@fmt//:fmt"],
-)
+)

bazel/leveldb.BUILD

@@ -0,0 +1,98 @@
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+package(default_visibility = ["//visibility:public"])
+
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+SOURCES = ["db/builder.cc",
+         "db/c.cc",
+         "db/dbformat.cc",
+         "db/db_impl.cc",
+         "db/db_iter.cc",
+         "db/dumpfile.cc",
+         "db/filename.cc",
+         "db/log_reader.cc",
+         "db/log_writer.cc",
+         "db/memtable.cc",
+         "db/repair.cc",
+         "db/table_cache.cc",
+         "db/version_edit.cc",
+         "db/version_set.cc",
+         "db/write_batch.cc",
+         "table/block_builder.cc",
+         "table/block.cc",
+         "table/filter_block.cc",
+         "table/format.cc",
+         "table/iterator.cc",
+         "table/merger.cc",
+         "table/table_builder.cc",
+         "table/table.cc",
+         "table/two_level_iterator.cc",
+         "util/arena.cc",
+         "util/bloom.cc",
+         "util/cache.cc",
+         "util/coding.cc",
+         "util/comparator.cc",
+         "util/crc32c.cc",
+         "util/env.cc",
+         "util/env_posix.cc",
+         "util/filter_policy.cc",
+         "util/hash.cc",
+         "util/histogram.cc",
+         "util/logging.cc",
+         "util/options.cc",
+         "util/status.cc",
+         "port/port_posix.cc",
+         "port/port_posix_sse.cc",
+         "helpers/memenv/memenv.cc",
+    ]
+
+cc_library(
+    name = "leveldb",
+    srcs = SOURCES,
+    hdrs = glob([ 
+        "helpers/memenv/*.h",
+        "util/*.h",
+        "port/*.h",
+        "port/win/*.h",
+        "table/*.h",
+        "db/*.h",
+        "include/leveldb/*.h"
+    ],
+    exclude = [
+            "**/*test.*",
+    ]),
+    includes = [
+        "include/",
+    ],
+    copts = [
+        "-fno-builtin-memcmp",
+        "-DLEVELDB_PLATFORM_POSIX=1",
+        "-DLEVELDB_ATOMIC_PRESENT",
+    ],
+    defines = [
+        "LEVELDB_PLATFORM_POSIX",
+    ] + select({
+        ":darwin": ["OS_MACOSX"],
+        "//conditions:default": [],
+    }),
+)

build.sh

@@ -51,16 +51,16 @@ curve_version=${tag_version}+${commit_id}${debug}
 
 
 #step3 执行编译
-# check bazel verion, bazel vesion must = 4.2.2
+# check bazel verion, bazel vesion must = 0.17.2
 bazel_version=`bazel version | grep "Build label" | awk '{print $3}'`
 if [ -z ${bazel_version} ]
 then
-    echo "please install bazel 4.2.2 first"
+    echo "please install bazel 0.17.2 first"
     exit
 fi
-if [ ${bazel_version} != "4.2.2" ]
+if [ ${bazel_version} != "0.17.2" ]
 then
-    echo "bazel version must 4.2.2"
+    echo "bazel version must 0.17.2"
     echo "now version is ${bazel_version}"
     exit
 fi

buildfs.sh

@@ -7,22 +7,12 @@ then
     exit
 fi
 
-if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
-then
-    bazelflags=''
-else
-    bazelflags='--copt -faligned-new'
-fi
-
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
---define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
--DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
-
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -44,5 +34,4 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
-echo "end compile"
+fi

conf/chunkserver.conf

@@ -9,20 +9,13 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
-global.block_size=4096
-
 # clone chunk允许的最长location长度
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512
 
 #
 # MDS settings
@@ -54,8 +47,6 @@ chunkserver.snapshot_throttle_throughput_bytes=20971520
 # 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
 # 不是20MB的带宽
 chunkserver.snapshot_throttle_check_cycles=4
-# 限制inflight io数量，一般是5000
-chunkserver.max_inflight_requests=5000
 
 #
 # Testing purpose settings
@@ -93,10 +84,9 @@ copyset.raft_meta_uri=local://./0/copysets  # __CURVEADM_TEMPLATE__ local://${pr
 copyset.raft_snapshot_uri=curve://./0/copysets  # __CURVEADM_TEMPLATE__ curve://${prefix}/data/copysets __CURVEADM_TEMPLATE__
 # copyset回收目录
 copyset.recycler_uri=local://./0/recycler  # __CURVEADM_TEMPLATE__ local://${prefix}/data/recycler __CURVEADM_TEMPLATE__
+copyset.max_inflight_requests=5000
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency=10
-# chunkserver use how many threads to use copyset complete sync. 
-copyset.sync_concurrency=20
 # 检查copyset是否加载完成出现异常时的最大重试次数
 copyset.check_retrytimes=3
 # 当前peer的applied_index与leader上的committed_index差距小于该值
@@ -114,16 +104,6 @@ copyset.scan_rpc_timeout_ms=1000
 copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
-# enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
-# sync trigger seconds
-copyset.sync_trigger_seconds=25
-# sync chunk limit default = 2MB
-copyset.sync_chunk_limits=2097152
-# 30s if the sum of write > sync_threshold, let the sync_chunk_limits doubled.
-copyset.sync_threshold=65536
-# check syncing interval
-copyset.check_syncing_interval_ms=500
 
 #
 # Clone settings
@@ -207,10 +187,8 @@ chunkfilepool.clean.throttle_iops=500
 #
 # WAL file pool
 #
-# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
+# walpool是否共用chunkfilepool，如果为true，则以下配置无效
 walfilepool.use_chunk_file_pool=true
-# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
-walfilepool.use_chunk_file_pool_reserve=15
 # 是否开启从walfilepool获取chunk，一般是true
 walfilepool.enable_get_segment_from_pool=true
 # walpool目录

conf/chunkserver.conf.example

@@ -13,12 +13,9 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-global.block_size=4096
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512
 
 #
 # MDS settings
@@ -50,8 +47,6 @@ chunkserver.snapshot_throttle_throughput_bytes=20971520
 # 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
 # 不是20MB的带宽
 chunkserver.snapshot_throttle_check_cycles=4
-# 限制inflight io数量，一般是5000
-chunkserver.max_inflight_requests=5000
 
 #
 # Testing purpose settings
@@ -89,10 +84,9 @@ copyset.raft_meta_uri=local://./0/copysets
 copyset.raft_snapshot_uri=curve://./0/copysets
 # copyset回收目录
 copyset.recycler_uri=local://./0/recycler
+copyset.max_inflight_requests=5000
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency=10
-# chunkserver use how many threads to use copyset complete sync. 
-copyset.sync_concurrency=20
 # 检查copyset是否加载完成出现异常时的最大重试次数
 copyset.check_retrytimes=3
 # 当前peer的applied_index与leader上的committed_index差距小于该值
@@ -110,16 +104,6 @@ copyset.scan_rpc_timeout_ms=1000
 copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
-# enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
-# sync trigger seconds
-copyset.sync_trigger_seconds=25
-# sync chunk limit default = 2MB
-copyset.sync_chunk_limits=2097152
-# 30s if the sum of write > sync_threshold, let the sync_chunk_limits doubled.
-copyset.sync_threshold=65536
-# check syncing interval
-copyset.check_syncing_interval_ms=500
 
 #
 # Clone settings
@@ -203,10 +187,8 @@ chunkfilepool.clean.throttle_iops=500
 #
 # WAL file pool
 #
-# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
+# walpool是否共用chunkfilepool，如果为true，则以下配置无效
 walfilepool.use_chunk_file_pool=true
-# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
-walfilepool.use_chunk_file_pool_reserve=15
 # 是否开启从walfilepool获取chunk，一般是true
 walfilepool.enable_get_segment_from_pool=true
 # walpool目录

conf/client.conf

@@ -129,9 +129,6 @@ global.fileIOSplitMaxSizeKB=64
 #
 ################# log相关配置 ###############
 #
-# enable logging or not
-global.logging.enable=True
-#
 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
 global.logLevel=0
 # 设置log的路径
@@ -168,16 +165,9 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
-##### chunkserver client option #####
-# chunkserver client rpc timeout time
-csClientOpt.rpcTimeoutMs=500
-# chunkserver client rpc max try
-csClientOpt.rpcMaxTry=86400000
-# chunkserver client rpc retry interval
-csClientOpt.rpcIntervalUs=100000
-# chunkserver client rpc max timeout time
-csClientOpt.rpcMaxTimeoutMs=8000
-
-##### chunkserver broadcaster option #####
-# broad cast max machine num
-csBroadCasterOpt.broadCastMaxNum=200
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/cs_client.conf

@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/mds.conf

@@ -156,10 +156,8 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 # 请求chunkserver上创建copyset重试间隔
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
-mds.topology.UpdateMetricIntervalSec=10
-#和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
-#那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
-#具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
+mds.topology.UpdateMetricIntervalSec=60
+# 物理池使用百分比，即使用量超过这个值即不再往这个池分配
 mds.topology.PoolUsagePercentLimit=85
 # 多pool选pool策略 0:Random, 1:Weight
 mds.topology.choosePoolPolicy=0
@@ -197,8 +195,6 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
-# smallest read/write unit for volume, support |512| and |4096|
-mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -239,16 +235,3 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
-
-#
-## poolset rules
-#
-# for backward compatibility, rules are applied for select poolset when creating file
-#
-# for example
-#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
-#
-# when creating file reqeust doesn't have poolset, above rules are used to select poolset
-# - if filename is /dir1/file, then poolset1 is select
-# - if filename is /dir1/sub/file, then sub is select
-mds.poolset.rules=

conf/py_client.conf

@@ -153,3 +153,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/s3.conf

@@ -7,20 +7,19 @@ s3.endpoint=
 # reserved for backward compatible
 s3.snapshot_bucket_name=
 s3.bucket_name=
-s3.ak=fake
-s3.sk=fake
-s3.region=us-east-1
+s3.ak=
+s3.sk=
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=false
 s3.user_agent_conf=S3 Browser
-s3.maxConnections=32
-s3.connectTimeout=60000
-s3.requestTimeout=10000
+s3.max_connections=32
+s3.connect_timeout=60000
+s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel=4
+s3.loglevel=4
 s3.logPrefix=/data/log/curve/aws_
-s3.asyncThreadNum=64
+s3.async_thread_num=64
 # throttle
 s3.throttle.iopsTotalLimit=5000
 s3.throttle.iopsReadLimit=5000
@@ -28,5 +27,3 @@ s3.throttle.iopsWriteLimit=5000
 s3.throttle.bpsTotalMB=1280
 s3.throttle.bpsReadMB=1280
 s3.throttle.bpsWriteMB=1280
-s3.useVirtualAddressing=false
-

conf/snap_client.conf

@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/tools.conf

@@ -11,8 +11,8 @@ rpcConcurrentNum=10
 # etcd地址
 etcdAddr=127.0.0.1:2379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__
 # snapshot clone server 地址
-snapshotCloneAddr=  # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_addr} __CURVEADM_TEMPLATE__
+snapshotCloneAddr=127.0.0.1:5555  # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_addr} __CURVEADM_TEMPLATE__
 # snapshot clone server dummy port
-snapshotCloneDummyPort=  # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_dummy_port} __CURVEADM_TEMPLATE__
+snapshotCloneDummyPort=8081  # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_dummy_port} __CURVEADM_TEMPLATE__
 rootUserName=root
 rootUserPassword=root_password

copts.bzl

@@ -120,8 +120,9 @@ CURVE_LLVM_FLAGS = [
     "-Wvla",
     "-Wwrite-strings",
     "-Wno-float-conversion",
-    "-Wno-float-conversion",
-    "-Wno-float-overflow-conversion",
+    "-Wno-implicit-float-conversion",
+    "-Wno-implicit-int-float-conversion",
+    "-Wno-implicit-int-conversion",
     "-Wno-shorten-64-to-32",
     "-Wno-sign-conversion",
     "-DNOMINMAX",
@@ -145,16 +146,10 @@ CURVE_LLVM_TEST_FLAGS = [
     "-Wno-used-but-marked-unused",
     "-Wno-zero-as-null-pointer-constant",
     "-Wno-gnu-zero-variadic-macro-arguments",
-    "-Wbraced-scalar-init",
-]
-
-# FIXME: temporary disabled because triggered in many places
-CURVE_LLVM_DISABLED_FLGAS = [
-    "-Wno-c++11-narrowing",
 ]
 
 CURVE_DEFAULT_COPTS = select({
-    "//:clang_compiler": CURVE_LLVM_FLAGS + CXX_FLAGS + BASE_FLAGS + CURVE_LLVM_DISABLED_FLGAS,
+    "//:clang_compiler": CURVE_LLVM_FLAGS + CXX_FLAGS + BASE_FLAGS,
     "//conditions:default": CURVE_GCC_FLAGS + CXX_FLAGS + BASE_FLAGS + CURVE_GCC_DISABLED_FLGAS,
 })
 

coverage/check_coverage.sh

@@ -0,0 +1,118 @@
+#!/bin/bash 
+
+line_cover_base=75
+mds_branch_base=70
+snapshot_branch_base=70
+client_branch_base=78
+other_branch_base=65
+line_cover_all=`cat coverage/index.html | grep -A 5  "Lines"  | grep % | awk -F "%" '{print $1}' | awk -F '>' '{print $2}' | awk -F '.' '{print $1}'`
+
+if(("$line_cover_all" < "$line_cover_base"))
+then
+    echo "line cover not ok!.";
+    echo $line_cover_all;
+    exit -1
+else
+    echo "line cover ok!.";
+    echo $line_cover_all;    
+fi
+
+for i in `find coverage -type d | grep mds`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > mds.all
+if [ -s mds.all ]; then    
+    mds_branch=`cat mds.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    mds_branch=0
+fi
+if(("$mds_branch" < "$mds_branch_base"))
+then
+    echo "mds_branch cover not ok!.";
+    echo $mds_branch;
+    exit -1
+else
+    echo "mds_branch cover ok!.";
+    echo $mds_branch;    
+fi
+
+for i in `find coverage -type d | grep tools`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > tools.all
+if [ -s tools.all ]; then
+    tools_branch=`cat tools.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    tools_branch=0
+fi
+
+if(("$tools_branch" < "$other_branch_base"))
+then
+    echo "tools_branch cover not ok!.";
+    echo $tools_branch;
+    #exit -1
+else
+    echo "tools_branch cover ok!.";
+    echo $tools_branch;    
+fi
+
+for i in `find coverage -type d | grep common`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > common.all
+if [ -s common.all ]; then
+    common_branch=`cat common.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    common_branch=0
+fi	
+if(("$common_branch" < "$other_branch_base"))
+then
+    echo "common_branch cover not ok!.";
+    echo $common_branch;
+    exit -1
+else
+    echo "common_branch cover ok!.";
+    echo $common_branch;    
+fi
+
+
+for i in `find coverage -type d | grep chunkserver`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > chunkserver.all
+if [ -s chunkserver.all ]; then
+    chunkserver_branch=`cat chunkserver.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    chunkserver_branch=0
+fi
+if(("$chunkserver_branch" < "$other_branch_base"))
+then
+    echo "chunkserver_branch cover not ok!.";
+    echo $chunkserver_branch;
+    exit -1
+else
+    echo "chunkserver_branch cover ok!.";
+    echo $chunkserver_branch;    
+fi
+
+
+for i in `find coverage/client -type d | grep client`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > client.all
+if [ -s chunkserver.all ]; then
+    client_branch=`cat client.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    client_branch=0
+fi
+if(("$client_branch" < "$client_branch_base"))
+then
+    echo "client_branch cover not ok!.";
+    echo $client_branch;
+    exit -1
+else
+    echo "client_branch cover ok!.";
+    echo $client_branch;    
+fi
+
+for i in `find coverage -type d | grep fs`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > sfs.all
+if [ -s sfs.all ]; then
+    sfs_branch=`cat sfs.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    sfs_branch=0
+fi
+if(("$sfs_branch" < "$other_branch_base"))
+then
+    echo "sfs_branch cover not ok!.";
+    echo $sfs_branch;
+    #exit -1
+else
+    echo "sfs_branch cover ok!.";
+    echo $sfs_branch;    
+fi
+

coverage/filterbr.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import sys
+
+def skip_comments(lines):
+  state = 0
+  for line in lines:
+    n = len(line)
+    l = ''
+    p = 0
+    while p < n:
+      if state == 0:
+        a = line.find('//', p)
+        b = line.find('/*', p)
+        if a > -1 and (a < b or b == -1):
+          l += line[p:a]
+          p = n
+        elif b > -1 and (b < a or a == -1):
+          l += line[p:b]
+          p = b+2
+          state = 1
+        else:
+          l += line[p:]
+          p = n
+      elif state == 1:
+        a = line.rfind('*/', p)
+        if a == -1:
+          p = n
+        else:
+          p = a + 2
+          state = 0
+    yield l
+
+def cond_lines(lines):
+  state = 0
+  pcnt = 0
+  for nr, line in enumerate(lines, 1):
+    if not line:
+      continue
+    n = len(line)
+    p = 0
+    do_yield = False
+    while p < n:
+      if state == 0:
+#        p = line.strip().startswith('if', p)
+        p = line.find('if', p)
+        if p == -1:
+          p = n
+          continue
+        if (p == 0 or not line[p-1].isalpha()) \
+            and (p+2 == len(line) or not line[p+2].isalpha()):
+          do_yield = True
+          state = 1
+        p += 2
+      elif state == 1:
+        do_yield = True
+        p = line.find('(', p)
+        if p == -1:
+          p = n
+        else:
+          p += 1
+          state = 2
+          pcnt = 1
+      elif state == 2:
+        do_yield = True
+        for p in range(p, n):
+          if line[p] == '(':
+            pcnt += 1
+          elif line[p] == ')':
+            pcnt -= 1
+          if not pcnt:
+            state = 0
+            break
+        p += 1
+    if do_yield:
+      yield nr
+
+def cond_lines_from_file(filename):
+  with open(filename) as f:
+    yield from cond_lines(skip_comments(f))
+
+def filter_lcov_trace(lines):
+  nrs = set()
+  for line in lines:
+    if line.startswith('SF:'):
+      nrs = set(cond_lines_from_file(line[3:-1]))
+    elif line.startswith('BRDA:'):
+      xs = line[5:].split(',')
+      nr = int(xs[0]) if xs else 0
+      if nr not in nrs:
+        continue
+    yield line
+
+def filter_lcov_trace_file(s_filename, d_file):
+  with open(s_filename) as f:
+    for l in filter_lcov_trace(f):
+      print(l, end='', file=d_file)
+
+if __name__ == '__main__':
+  #for l in cond_lines_from_file(sys.argv[1]):
+  #  print(l)
+
+  filter_lcov_trace_file(sys.argv[1], sys.stdout)
+
+  #with open(sys.argv[1]) as f:
+  #  for l in skip_comments(f):
+  #    print(l)
+
+

coverage/gen-coverage-nebd.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import filterbr
+
+log = logging.getLogger(__name__)
+
+#ex_path = [ '/usr/include/*', 'unittest/*', 'lib*/*', '*@exe/*', 'example/*' ]
+ex_path = [ '/usr/include/*', '/usr/local/include/*', '/usr/lib/*', '*/bazel_out/*', '*/k8-dbg/*', 'test/*', '*/test/*', '*/external/*' , '*/include/*' , '*/thirdparties/*' , '*/client_proto/*' ]
+
+brflag = ['--rc', 'lcov_branch_coverage=1']
+
+lcov = 'lcov'
+base = os.path.abspath('.')
+
+cov_init_raw = 'coverage_init_raw.info'
+cov_post_raw = 'coverage_post_raw.info'
+cov_init     = 'coverage_init.info'
+cov_post     = 'coverage_post.info'
+cov_br       = 'coverage.info'
+cov          = 'coverage.info'
+report_dir   = 'coverage'
+
+def setup_logging():
+  log_format      = '{rel_secs:6.1f} {lvl}  {message}'
+  log_date_format = '%Y-%m-%d %H:%M:%S'
+
+  class Relative_Formatter(logging.Formatter):
+    level_dict = { 10 : 'DBG',  20 : 'INF', 30 : 'WRN', 40 : 'ERR',
+        50 : 'CRI' }
+    def format(self, rec):
+      rec.rel_secs = rec.relativeCreated/1000.0
+      rec.lvl = self.level_dict[rec.levelno]
+      return super(Relative_Formatter, self).format(rec)
+
+  log = logging.getLogger() # root logger
+  log.setLevel(logging.DEBUG)
+  ch = logging.StreamHandler()
+  ch.setLevel(logging.INFO)
+  ch.setFormatter(Relative_Formatter(log_format, log_date_format, style='{'))
+  log.addHandler(ch)
+
+def mk_arg_parser():
+  p = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Do some stuff',
+        epilog='...')
+  p.add_argument('--html', action='store_true', default=True,
+      help='generate HTML report (default: on)')
+  p.add_argument('--no-html', dest='html', action='store_false',
+      help='disable html report generation')
+  p.add_argument('--filter-br', action='store_true', default=True,
+      help='filter branch coverage data (default: on)')
+  p.add_argument('--no-filter-br', dest='filter_br', action='store_false',
+      help='disable branch filtering')
+  return p
+
+def parse_args(*a):
+  arg_parser = mk_arg_parser()
+  args = arg_parser.parse_args(*a)
+  global cov_br
+  if args.filter_br:
+    cov_br = 'coverage_br.info'
+  return args
+
+def run(*args, **kw):
+  log.info('Executing: ' + ' '.join(map(lambda s:"'"+s+"'", args[0])))
+  return subprocess.run(*args, **kw, check=True)
+
+
+def main(args):
+  run([lcov, '--directory',  'src', '--capture', '-o', cov_post_raw] + brflag )
+  run([lcov, '--directory',  'src', '--capture', '--initial', '-o', cov_init_raw])
+
+  for i, o in [ (cov_init_raw, cov_init), (cov_post_raw, cov_post) ]:
+    run([lcov, '--remove', i] + ex_path + [ '-o', o] + brflag)
+
+  run([lcov, '-a', cov_init, '-a', cov_post, '-o', cov_br] + brflag)
+
+  if args.filter_br:
+    log.info('Filtering branch coverage data ({} -> {})'.format(cov_br, cov))
+    with open(cov, 'w') as f:
+      filterbr.filter_lcov_trace_file(cov_br, f)
+
+  if args.html:
+    shutil.rmtree(report_dir, ignore_errors=True)
+    run(['genhtml', cov, '--branch-coverage', '--ignore-errors', 'source', '-o', report_dir])
+
+  return 0
+
+if __name__ == '__main__':
+  setup_logging()
+  args = parse_args()
+  sys.exit(main(args))
+
+
+

coverage/gen-coverage.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import filterbr
+
+log = logging.getLogger(__name__)
+
+#ex_path = [ '/usr/include/*', 'unittest/*', 'lib*/*', '*@exe/*', 'example/*' ]
+ex_path = [ '/usr/include/*', '/usr/local/include/*', '/usr/lib/*', '*/bazel_out/*', '*/k8-dbg/*', 'test/*', '*/test/*', '*/external/*' , '*/include/*' , '*/thirdparties/*' ]
+
+brflag = ['--rc', 'lcov_branch_coverage=1']
+
+lcov = 'lcov'
+base = os.path.abspath('.')
+
+cov_init_raw = 'coverage_init_raw.info'
+cov_post_raw = 'coverage_post_raw.info'
+cov_init     = 'coverage_init.info'
+cov_post     = 'coverage_post.info'
+cov_br       = 'coverage.info'
+cov          = 'coverage.info'
+report_dir   = 'coverage'
+
+def setup_logging():
+  log_format      = '{rel_secs:6.1f} {lvl}  {message}'
+  log_date_format = '%Y-%m-%d %H:%M:%S'
+
+  class Relative_Formatter(logging.Formatter):
+    level_dict = { 10 : 'DBG',  20 : 'INF', 30 : 'WRN', 40 : 'ERR',
+        50 : 'CRI' }
+    def format(self, rec):
+      rec.rel_secs = rec.relativeCreated/1000.0
+      rec.lvl = self.level_dict[rec.levelno]
+      return super(Relative_Formatter, self).format(rec)
+
+  log = logging.getLogger() # root logger
+  log.setLevel(logging.DEBUG)
+  ch = logging.StreamHandler()
+  ch.setLevel(logging.INFO)
+  ch.setFormatter(Relative_Formatter(log_format, log_date_format, style='{'))
+  log.addHandler(ch)
+
+def mk_arg_parser():
+  p = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Do some stuff',
+        epilog='...')
+  p.add_argument('--html', action='store_true', default=True,
+      help='generate HTML report (default: on)')
+  p.add_argument('--no-html', dest='html', action='store_false',
+      help='disable html report generation')
+  p.add_argument('--filter-br', action='store_true', default=True,
+      help='filter branch coverage data (default: on)')
+  p.add_argument('--no-filter-br', dest='filter_br', action='store_false',
+      help='disable branch filtering')
+  return p
+
+def parse_args(*a):
+  arg_parser = mk_arg_parser()
+  args = arg_parser.parse_args(*a)
+  global cov_br
+  if args.filter_br:
+    cov_br = 'coverage_br.info'
+  return args
+
+def run(*args, **kw):
+  log.info('Executing: ' + ' '.join(map(lambda s:"'"+s+"'", args[0])))
+  return subprocess.run(*args, **kw, check=True)
+
+
+def main(args):
+  run([lcov, '--directory',  'src', '--capture', '-o', cov_post_raw] + brflag )
+  run([lcov, '--directory',  'src', '--capture', '--initial', '-o', cov_init_raw])
+
+  for i, o in [ (cov_init_raw, cov_init), (cov_post_raw, cov_post) ]:
+    run([lcov, '--remove', i] + ex_path + [ '-o', o] + brflag)
+
+  run([lcov, '-a', cov_init, '-a', cov_post, '-o', cov_br] + brflag)
+
+  if args.filter_br:
+    log.info('Filtering branch coverage data ({} -> {})'.format(cov_br, cov))
+    with open(cov, 'w') as f:
+      filterbr.filter_lcov_trace_file(cov_br, f)
+
+  if args.html:
+    shutil.rmtree(report_dir, ignore_errors=True)
+    run(['genhtml', cov, '--branch-coverage', '--ignore-errors', 'source', '-o', report_dir])
+
+  return 0
+
+if __name__ == '__main__':
+  setup_logging()
+  args = parse_args()
+  sys.exit(main(args))
+
+
+

coverage/gen-coverage2.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import filterbr
+
+log = logging.getLogger(__name__)
+
+#ex_path = [ '/usr/include/*', 'unittest/*', 'lib*/*', '*@exe/*', 'example/*' ]
+ex_path = [ '/usr/include/*', '/usr/local/include/*', '/usr/lib/*', '*/bazel_out/*', '*/k8-dbg/*', 'test/*', '*/test/*', '*/external/*' , '*/include/*' , '*/thirdparties/*' , '*/snapshotcloneserver/*' ]
+
+brflag = ['--rc', 'lcov_branch_coverage=1']
+
+lcov = 'lcov'
+base = os.path.abspath('.')
+
+cov_init_raw = 'coverage_init_raw.info'
+cov_post_raw = 'coverage_post_raw.info'
+cov_init     = 'coverage_init.info'
+cov_post     = 'coverage_post.info'
+cov_br       = 'coverage.info'
+cov          = 'coverage.info'
+report_dir   = 'coverage'
+
+def setup_logging():
+  log_format      = '{rel_secs:6.1f} {lvl}  {message}'
+  log_date_format = '%Y-%m-%d %H:%M:%S'
+
+  class Relative_Formatter(logging.Formatter):
+    level_dict = { 10 : 'DBG',  20 : 'INF', 30 : 'WRN', 40 : 'ERR',
+        50 : 'CRI' }
+    def format(self, rec):
+      rec.rel_secs = rec.relativeCreated/1000.0
+      rec.lvl = self.level_dict[rec.levelno]
+      return super(Relative_Formatter, self).format(rec)
+
+  log = logging.getLogger() # root logger
+  log.setLevel(logging.DEBUG)
+  ch = logging.StreamHandler()
+  ch.setLevel(logging.INFO)
+  ch.setFormatter(Relative_Formatter(log_format, log_date_format, style='{'))
+  log.addHandler(ch)
+
+def mk_arg_parser():
+  p = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Do some stuff',
+        epilog='...')
+  p.add_argument('--html', action='store_true', default=True,
+      help='generate HTML report (default: on)')
+  p.add_argument('--no-html', dest='html', action='store_false',
+      help='disable html report generation')
+  p.add_argument('--filter-br', action='store_true', default=True,
+      help='filter branch coverage data (default: on)')
+  p.add_argument('--no-filter-br', dest='filter_br', action='store_false',
+      help='disable branch filtering')
+  return p
+
+def parse_args(*a):
+  arg_parser = mk_arg_parser()
+  args = arg_parser.parse_args(*a)
+  global cov_br
+  if args.filter_br:
+    cov_br = 'coverage_br.info'
+  return args
+
+def run(*args, **kw):
+  log.info('Executing: ' + ' '.join(map(lambda s:"'"+s+"'", args[0])))
+  return subprocess.run(*args, **kw, check=True)
+
+
+def main(args):
+  run([lcov, '--directory',  'src', '--capture', '-o', cov_post_raw] + brflag )
+  run([lcov, '--directory',  'src', '--capture', '--initial', '-o', cov_init_raw])
+
+  for i, o in [ (cov_init_raw, cov_init), (cov_post_raw, cov_post) ]:
+    run([lcov, '--remove', i] + ex_path + [ '-o', o] + brflag)
+
+  run([lcov, '-a', cov_init, '-a', cov_post, '-o', cov_br] + brflag)
+
+  if args.filter_br:
+    log.info('Filtering branch coverage data ({} -> {})'.format(cov_br, cov))
+    with open(cov, 'w') as f:
+      filterbr.filter_lcov_trace_file(cov_br, f)
+
+  if args.html:
+    shutil.rmtree(report_dir, ignore_errors=True)
+    run(['genhtml', cov, '--branch-coverage', '--ignore-errors', 'source', '-o', report_dir])
+
+  return 0
+
+if __name__ == '__main__':
+  setup_logging()
+  args = parse_args()
+  sys.exit(main(args))
+
+
+

coverage/ut_incremental_check.py

@@ -0,0 +1,226 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+######################################################################
+# Purpose:    calculate UT coverage of git commits' new code
+# Useage:    ./ut_incremental_check.py
+# Version:    Initial    Version    by wahaha02
+######################################################################
+
+__version__ = 'V1.0'
+__author__ = 'wahaha02'
+__date__ = '2016-7-25'
+__doc__ = '''
+PURPOSE:
+calculate UT coverage of git commits' new code
+
+USAGE:
+./ut_incremental_check.py <since>..<until> <monitor_c_files> <lcov_dir> <threshold>
+example: 
+./ut_incremental_check.py "227b032..79196ba" '["source/soda/sp/lssp/i2c-v2/ksource"]' "coverage" 0.6
+
+WORK PROCESS:
+get changed file list between <since> and <until> , filter by <monitor_c_files> options;
+get changed lines per changed file;
+based on <lcov_dir>, search .gcov.html per file, and get uncover lines;
+create report file:ut_incremental_check_report.html and check <threshold> (cover lines/new lines).
+
+UT:
+./ut_incremental_check.py ut
+'''
+
+__todo__ = '''
+TODO LIST:
+    1. support svn
+    2. refactory html report by django web template
+    3. add commit info in html report
+    4. prompt user/commit/date info when mouse point to uncovered line
+    5. ...
+'''
+
+import sys, os, re
+import json
+import commands
+from HTMLParser import HTMLParser
+from pprint import *
+ 
+DEBUG = 0 
+
+class GcovHTMLParser(HTMLParser):
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.uncovers = []
+        self.covers = []
+        self.islineNum = False
+        self.lineNum = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "span":
+            for a in attrs:
+                if a == ('class', 'lineNum'):
+                    self.islineNum = True
+                if a == ('class', 'lineNoCov'):
+                    self.uncovers.append(self.lineNum)
+                if a == ('class', 'lineCov'):
+                    self.covers.append(self.lineNum)
+                    
+    def handle_data(self, data):
+        if self.islineNum:
+            try:
+                self.lineNum = int(data)
+            except:
+                self.lineNum = -1
+            
+    def handle_endtag(self, tag):
+        if tag == "span":
+            self.islineNum = False
+    
+class UTCover(object) :
+    def __init__(self, since_until, monitor, lcov_dir, thresh) :
+        self.since, self.until = since_until.split('..')
+        self.monitor = json.loads(monitor)
+        self.lcov_dir = lcov_dir
+        self.thresh = float(thresh)
+        
+    def get_src(self):
+        # self.since, self.until, self.monitor
+        satus, output = commands.getstatusoutput("git diff --name-only %s %s" %(self.since, self.until))
+        src_files = [f for f in output.split('\n') 
+                        for m in self.monitor if m in f 
+                        if os.path.splitext(f)[1][1:] in ['c', 'cpp']]
+        if DEBUG: pprint(src_files)
+        for i in src_files:
+            if i.find("main") > 0:
+                src_files.remove(i)
+        return src_files
+    
+    def get_change(self, src_files):
+        # self.since, self.until
+        changes = {}
+        for f in src_files:
+            satus, output = commands.getstatusoutput("git log --oneline %s..%s %s | awk '{print $1}'" %(self.since, self.until, f))
+            commits = output.split('\n')
+            cmd = "git blame %s | grep -E '(%s)' | awk  -F' *|)' '{print $6}'" %(f, '|'.join(commits))
+            satus, lines = commands.getstatusoutput(cmd)
+            changes[f] = [ int(i) for i in lines.split('\n') if i.isdigit() ]
+
+            
+        if DEBUG: pprint(changes)
+        return changes
+    
+    def get_ghp(self, f):
+        gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+        if not os.path.exists(gcovfile):
+            return None
+            
+        ghp = GcovHTMLParser()
+        ghp.feed(open(gcovfile, 'r').read())
+
+        return ghp
+            
+    def get_lcov_data(self, changes):
+        # self.lcov_dir
+        uncovers = {}
+        lcov_changes = {}
+        
+        for f, lines in changes.items():
+            #print f
+            f = f.split('/', 1)[1]
+            #f = [c.split('/', 1)[1] for c in f]
+            ghp = self.get_ghp(f)
+            if not ghp:
+                uncovers[f] = lines
+                lcov_changes[f] = lines
+                continue
+        
+            if DEBUG: print f, ghp.uncovers, ghp.covers, lines
+            lcov_changes[f] = sorted(list(set(ghp.uncovers + ghp.covers) & set(lines)))
+            uncov_lines = list(set(ghp.uncovers) & set(lines))
+            if len(uncov_lines) != 0:
+                uncovers[f] = sorted(uncov_lines)
+            ghp.close()    
+        
+        return lcov_changes, uncovers
+    
+    def create_uncover_trs(self, uncovers):
+        tr_format = '''
+    <tr>
+      <td class="coverFile"><a href="%(file)s.gcov.html">%(file)s</a></td>
+      <td class="coverFile">%(uncov_lines)s </td>
+    </tr>
+    
+        '''
+        trs = ''
+        for f,v in uncovers.items():
+            gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+            if os.path.exists(gcovfile):
+                s = ''
+                p = re.compile(r'^<span class="lineNum">\s*(?P<num>\d+)\s*</span>')
+                for line in open(gcovfile, 'r').readlines():
+                    ps = p.search(line)
+                    if ps:
+                        s += '<a name="%s">' %ps.group('num') + line + '</a>'
+                    else:
+                        s += line
+                open(gcovfile, 'w').write(s)    
+
+            data = {'file':f, 'uncov_lines':
+                ", ".join(['<a href="%s.gcov.html#%d">%d</a>' %(f, i, i) for i in v])}
+            trs += tr_format %data
+            
+        return trs
+        
+    def create_report(self, changes, uncovers):
+        change_linenum, uncov_linenum = 0, 0
+        for k,v in changes.items():
+            change_linenum += len(v)
+        for k,v in uncovers.items():
+            uncov_linenum += len(v)
+        
+        cov_linenum = change_linenum - uncov_linenum
+        coverage = round(cov_linenum * 1.0 / change_linenum 
+                            if change_linenum > 0 else 1, 4)
+        
+        template = open('ut_incremental_coverage_report.template', 'r').read()
+        data = {    'cov_lines':cov_linenum,
+                    'change_linenum':change_linenum,
+                    'coverage': coverage * 100,
+                    'uncover_trs': self.create_uncover_trs(uncovers)}
+        open(os.path.join(self.lcov_dir, 'ut_incremental_coverage_report.html'),
+            'w').write(template %data)    
+        
+        return coverage
+    
+    def check(self):
+        # main function
+        src_files = self.get_src()
+        changes = self.get_change(src_files)
+        lcov_changes, uncovers = self.get_lcov_data(changes)
+        return 0 if self.create_report(lcov_changes, uncovers) > self.thresh else -1 
+        
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(0)
+if sys.argv[1] == 'ut':
+    monitor, lcov_dir, threshold = ['["src/"]', "/root/.cache/bazel/_bazel_root/6f45ce5b16fddc736efcfa6b0a4c96a2/execroot/curve/bazel-out/k8-dbg/bin/coverage", 0.95]
+    test1 = ["960fc84f..9a928260", monitor, lcov_dir, threshold]
+    if DEBUG: print "test1: ", test1
+    ut = UTCover(*test1)
+    src_files = ut.get_src()
+    print src_files
+    print "====================="
+    changes = ut.get_change(src_files)
+    print "====================="
+    lcov_changes, uncovers = ut.get_lcov_data(changes)
+    print "====================="
+    print uncovers
+    print "====================="
+    rate = ut.create_report(changes, uncovers)
+    print rate
+    print "===================="
+    
+    
+    sys.exit(0)
+    
+ret = UTCover(*sys.argv[1:]).check()
+sys.exit(ret)
+

coverage/ut_incremental_check_nebd.py

@@ -0,0 +1,252 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+######################################################################
+# Purpose:    calculate UT coverage of git commits' new code
+# Useage:    ./ut_incremental_check.py
+# Version:    Initial    Version    by wahaha02
+######################################################################
+
+__version__ = 'V1.0'
+__author__ = 'wahaha02'
+__date__ = '2016-7-25'
+__doc__ = '''
+PURPOSE:
+calculate UT coverage of git commits' new code
+
+USAGE:
+./ut_incremental_check.py <since>..<until> <monitor_c_files> <lcov_dir> <threshold>
+example: 
+./ut_incremental_check.py "227b032..79196ba" '["source/soda/sp/lssp/i2c-v2/ksource"]' "coverage" 0.6
+
+WORK PROCESS:
+get changed file list between <since> and <until> , filter by <monitor_c_files> options;
+get changed lines per changed file;
+based on <lcov_dir>, search .gcov.html per file, and get uncover lines;
+create report file:ut_incremental_check_report.html and check <threshold> (cover lines/new lines).
+
+UT:
+./ut_incremental_check.py ut
+'''
+
+__todo__ = '''
+TODO LIST:
+    1. support svn
+    2. refactory html report by django web template
+    3. add commit info in html report
+    4. prompt user/commit/date info when mouse point to uncovered line
+    5. ...
+'''
+
+import sys, os, re
+import json
+import commands
+from HTMLParser import HTMLParser
+from pprint import *
+ 
+DEBUG = 0
+
+class GcovHTMLParser(HTMLParser):
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.uncovers = []
+        self.covers = []
+        self.islineNum = False
+        self.lineNum = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "span":
+            for a in attrs:
+                if a == ('class', 'lineNum'):
+                    self.islineNum = True
+                if a == ('class', 'lineNoCov'):
+                    self.uncovers.append(self.lineNum)
+                if a == ('class', 'lineCov'):
+                    self.covers.append(self.lineNum)
+                    
+    def handle_data(self, data):
+        if self.islineNum:
+            try:
+                self.lineNum = int(data)
+            except:
+                self.lineNum = -1
+            
+    def handle_endtag(self, tag):
+        if tag == "span":
+            self.islineNum = False
+    
+class UTCover(object) :
+    def __init__(self, since_until, monitor, lcov_dir, thresh) :
+        self.since, self.until = since_until.split('..')
+        self.monitor = json.loads(monitor)
+        self.lcov_dir = lcov_dir
+        self.thresh = float(thresh)
+        
+    def get_src(self):
+        # self.since, self.until, self.monitor
+        satus, output = commands.getstatusoutput("git diff --name-only %s %s" %(self.since, self.until))
+        src_files = [f for f in output.split('\n') 
+                        for m in self.monitor if m in f 
+                        if os.path.splitext(f)[1][1:] in ['c', 'cpp', 'cc']]
+        exclude = ['src/test']
+        src_files_clean = []
+        for f in src_files:
+            need_excluded = False
+            for e in exclude:
+                if e in f:
+                    need_excluded = True
+                    break
+            if not need_excluded:
+                src_files_clean.append(f)
+        print src_files_clean
+        if DEBUG: pprint(src_files)
+        return src_files_clean
+    
+    def get_change(self, src_files):
+        # self.since, self.until
+        changes = {}
+        for f in src_files:
+            satus, output = commands.getstatusoutput("git log --oneline %s..%s %s | awk '{print $1}'" %(self.since, self.until, f))
+            commits = output.split('\n')
+            cmd = "git blame %s | grep -E '(%s)' | cut -d\) -f1 | awk '{print $NF}'" %(f, '|'.join(commits))
+            print cmd
+            satus, lines = commands.getstatusoutput(cmd)
+            changes[f.split("src/")[1]] = [ int(i) for i in lines.split('\n') if i.isdigit() ]
+
+            
+        if DEBUG: pprint(changes)
+        print changes
+        return changes
+    
+    def get_ghp(self, f):
+        gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+        if not os.path.exists(gcovfile):
+            print "%s does not exist" % gcovfile
+            return None
+            
+        ghp = GcovHTMLParser()
+        ghp.feed(open(gcovfile, 'r').read())
+
+        return ghp
+            
+    def get_lcov_data(self, changes):
+        # self.lcov_dir
+        uncovers = {}
+        lcov_changes = {}
+        
+        for f, lines in changes.items():
+            ghp = self.get_ghp(f)
+            if not ghp:
+                uncovers[f] = lines
+                lcov_changes[f] = lines
+                continue
+        
+            if DEBUG: print f, ghp.uncovers, ghp.covers, lines
+            lcov_changes[f] = sorted(list(set(ghp.uncovers + ghp.covers) & set(lines)))
+            uncov_lines = list(set(ghp.uncovers) & set(lines))
+            if len(uncov_lines) != 0:
+                uncovers[f] = sorted(uncov_lines)
+            ghp.close()    
+        
+        return lcov_changes, uncovers
+    
+    def create_uncover_trs(self, uncovers):
+        tr_format = '''
+    <tr>
+      <td class="coverFile"><a href="%(file)s.gcov.html">%(file)s</a></td>
+      <td class="coverFile">%(uncov_lines)s </td>
+    </tr>
+    
+        '''
+        trs = ''
+        for f,v in uncovers.items():
+            gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+            if os.path.exists(gcovfile):
+                s = ''
+                p = re.compile(r'^<span class="lineNum">\s*(?P<num>\d+)\s*</span>')
+                for line in open(gcovfile, 'r').readlines():
+                    ps = p.search(line)
+                    if ps:
+                        s += '<a name="%s">' %ps.group('num') + line + '</a>'
+                    else:
+                        s += line
+                open(gcovfile, 'w').write(s)    
+
+            data = {'file':f, 'uncov_lines':
+                ", ".join(['<a href="%s.gcov.html#%d">%d</a>' %(f, i, i) for i in v])}
+            trs += tr_format %data
+            
+        return trs
+        
+    def create_report(self, changes, uncovers):
+        change_linenum, uncov_linenum = 0, 0
+        for k,v in changes.items():
+            change_linenum += len(v)
+        for k,v in uncovers.items():
+            uncov_linenum += len(v)
+        
+        cov_linenum = change_linenum - uncov_linenum
+        coverage = round(cov_linenum * 1.0 / change_linenum 
+                            if change_linenum > 0 else 1, 4)
+        
+        template = open('ut_incremental_coverage_report.template', 'r').read()
+        data = {    'cov_lines':cov_linenum,
+                    'change_linenum':change_linenum,
+                    'coverage': coverage * 100,
+                    'uncover_trs': self.create_uncover_trs(uncovers)}
+        open(os.path.join(self.lcov_dir, 'ut_incremental_coverage_report.html'),
+            'w').write(template %data)    
+        
+        return coverage
+    
+    def check(self):
+        # main function
+        src_files = self.get_src()
+        changes = self.get_change(src_files)
+        lcov_changes, uncovers = self.get_lcov_data(changes)
+        return 0 if self.create_report(lcov_changes, uncovers) > self.thresh else 1
+        
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(0)
+if sys.argv[1] == 'ut':
+    monitor, lcov_dir, threshold = ['["source/soda/sp/lssp/i2c-v2/ksource"]', "coverage", 0.8]
+    test1 = ["b2016fdb..11440652", monitor, lcov_dir, threshold]
+    if DEBUG: print "test1: ", test1
+    ut = UTCover(*test1)
+    src_files = ut.get_src()
+    assert(src_files == [])
+    changes = ut.get_change(src_files)
+    assert(changes == {})
+    lcov_changes, uncovers = ut.get_lcov_data(changes)
+    assert(uncovers == {})
+    rate = ut.create_report(changes, uncovers)
+    assert(rate == 1)
+    assert(ut.check() == 0)
+    
+    test2 = [
+        "227b03259b33360e2309274f3927c38457d84dd3..79196baabed99661bd31a201ead6764f23a2884c", 
+        monitor, lcov_dir, threshold]
+    if DEBUG: print "test2: ", test2
+    ut = UTCover(*test2)
+    src_files = ut.get_src()
+    assert(src_files == ['source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c', 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c', 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c', 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c'])
+    changes = ut.get_change(src_files)
+    assert(changes ==  {'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c': [78], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c': [56, 57, 58, 59, 60, 130, 131, 132], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c': [68, 69, 115, 118, 124, 125, 126, 454, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 471, 721], 'source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c': [494, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652]})
+    lcov_changes, uncovers = ut.get_lcov_data(changes)
+    assert( lcov_changes == {'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c': [78], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c': [56, 57, 58, 59, 60, 130, 131, 132], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c': [125, 459, 461, 462, 471], 'source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c': [496, 498, 502, 503, 504, 625, 629, 630, 631, 633, 634, 636, 638, 639, 643, 644, 649, 650]}) 
+    assert(uncovers == {'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c': [78], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c': [56, 57, 58, 59, 60, 130, 131, 132], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c': [125, 471], 'source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c': [502, 503, 504, 643, 644]})
+    rate = ut.create_report(changes, uncovers)
+    assert(0.8 > rate > 0.6)
+    assert(ut.check() == 1)
+    
+    test3 = ['d98b93e705a227389e7cdc4b43252f4194a6cb7a..e8876ff5fe8ee0e61865315a67bd395f5d7f63f7 ',
+            monitor, lcov_dir, threshold]
+    if DEBUG: print "test3: ", test3
+    ut = UTCover(*test3)
+    assert(ut.check() == 0)    
+    
+    sys.exit(0)
+    
+ret = UTCover(*sys.argv[1:]).check()
+sys.exit(ret)
+

curve-ansible/README.md

@@ -1,5 +1,3 @@
-**curve-ansile项目当前已经不进行维护，请使用最新工具curveadm进行部署：https://github.com/opencurve/curveadm**
-
 curve-ansible是用ansible编写的curve高性能存储系统远程部署升级工具，可以很方便的在一台主控机上做到部署升级集群所有节点。
 
 ##  一、软件依赖

curve-ansible/roles/generate_config/defaults/main.yml

@@ -105,7 +105,6 @@ chunkserver_heartbeat_timeout: 5000
 chunkserver_stor_uri: local://./0/
 chunkserver_meta_uri: local://./0/chunkserver.dat
 chunkserver_disk_type: nvme
-chunkserver_max_inflight_requests: 5000
 chunkserver_snapshot_throttle_throughput_bytes: 20971520
 chunkserver_snapshot_throttle_check_cycles: 4
 chunkserver_test_create_testcopyset: false
@@ -123,6 +122,7 @@ chunkserver_copyset_raft_log_uri: curve://./0/copysets
 chunkserver_copyset_raft_meta_uri: local://./0/copysets
 chunkserver_copyset_raft_snapshot_uri: curve://./0/copysets
 chunkserver_copyset_recycler_uri: local://./0/recycler
+chunkserver_copyset_max_inflight_requests: 5000
 chunkserver_copyset_load_concurrency: 10
 chunkserver_copyset_check_retrytimes: 3
 chunkserver_copyset_finishload_margin: 2000
@@ -132,9 +132,6 @@ chunkserver_copyset_scan_size_byte: 4194304
 chunkserver_copyset_scan_rpc_timeout_ms: 1000
 chunkserver_copyset_scan_rpc_retry_times: 3
 chunkserver_copyset_scan_rpc_retry_interval_us: 100000
-chunkserver_copyset_enable_odsync_when_open_chunkfile: false
-chunkserver_copyset_synctimer_interval_ms: 30000
-chunkserver_copyset_check_syncing_interval_ms: 500
 chunkserver_clone_slice_size: 1048576
 chunkserver_clone_enable_paste: false
 chunkserver_clone_thread_num: 10
@@ -164,6 +161,7 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
+chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -243,6 +241,8 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
+client_alignment_common: 512
+client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,6 +14,8 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
+# minimum alignment for io request
+global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings
@@ -51,7 +53,6 @@ chunkserver.snapshot_throttle_throughput_bytes={{ chunkserver_snapshot_throttle_
 # 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
 # 不是20MB的带宽
 chunkserver.snapshot_throttle_check_cycles={{ chunkserver_snapshot_throttle_check_cycles }}
-chunkserver.max_inflight_requests={{ chunkserver_max_inflight_requests }}
 
 #
 # Testing purpose settings
@@ -89,6 +90,7 @@ copyset.raft_meta_uri={{ chunkserver_copyset_raft_meta_uri }}
 copyset.raft_snapshot_uri={{ chunkserver_copyset_raft_snapshot_uri }}
 # copyset回收目录
 copyset.recycler_uri={{ chunkserver_copyset_recycler_uri }}
+copyset.max_inflight_requests={{ chunkserver_copyset_max_inflight_requests }}
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency={{ chunkserver_copyset_load_concurrency }}
 # 检查copyset是否加载完成出现异常时的最大重试次数
@@ -108,9 +110,6 @@ copyset.scan_rpc_timeout_ms={{ chunkserver_copyset_scan_rpc_timeout_ms }}
 copyset.scan_rpc_retry_times={{ chunkserver_copyset_scan_rpc_retry_times }}
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us={{ chunkserver_copyset_scan_rpc_retry_interval_us }}
-copyset.enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
-copyset.synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
-copyset.check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
 
 #
 # Clone settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,3 +175,10 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume={{ client_alignment_common }}
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume={{ client_alignment_clone }}

curve-ansible/roles/generate_config/templates/grafana.ini.j2

@@ -1,3 +1,21 @@
+#
+#     Copyright (c) 2020 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
 ##################### Grafana Configuration Example #####################
 #
 # Everything has defaults so you only need to uncomment things you want to

curve-ansible/roles/generate_config/templates/prometheus.yml.j2

@@ -1,3 +1,21 @@
+#
+#     Copyright (c) 2020 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
 # my global config
 global:
   scrape_interval:     {{ prometheus_scrape_interval }} # Set the scrape interval to every 15 seconds. Default is every 1 minute.

curve-ansible/roles/generate_config/templates/s3.conf.j2

@@ -11,13 +11,13 @@ s3.sk={{ s3_sk }}
 s3.http_scheme={{ s3_http_scheme }}
 s3.verify_SSL={{ s3_verify_ssl }}
 s3.user_agent_conf={{ s3_user_agent_conf }}
-s3.maxConnections={{ s3_max_connections }}
-s3.connectTimeout={{ s3_connect_timeout }}
-s3.requestTimeout={{ s3_request_timeout }}
+s3.max_connections={{ s3_max_connections }}
+s3.connect_timeout={{ s3_connect_timeout }}
+s3.request_timeout={{ s3_request_timeout }}
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel={{ s3_loglevel }}
+s3.loglevel={{ s3_loglevel }}
 s3.logPrefix={{ s3_logPrefix }}
-s3.asyncThreadNum={{ s3_async_thread_num }}
+s3.async_thread_num={{ s3_async_thread_num }}
 # throttle
 s3.throttle.iopsTotalLimit={{ s3_throttle_iopsTotalLimit }}
 s3.throttle.iopsReadLimit={{ s3_throttle_iopsReadLimit }}

curve-ansible/roles/install_package/tasks/include/install_curve-nbd.yml

@@ -73,5 +73,5 @@
             file_mode: 0755
           include_tasks: copy_file_to_remote.yml
         - name: enable service
-          shell: sudo systemctl daemon-reload && sudo systemctl enable map_curve_disk.service
+          shell: sudo systemctl daemon-reload && sudo systemctl enable map_curve_disk.service && sudo systemctl start map_curve_disk.service
   when: not install_with_deb | bool

curve-snapshotcloneserver-nginx/app/etc/config.lua

@@ -0,0 +1,53 @@
+local _M = {}
+
+-- check all the configure from
+-- https://github.com/upyun/lua-resty-checkups
+
+_M.global = {
+    --  Interval of sending heartbeats to backend servers.
+    checkup_timer_interval = 15,
+    
+    -- If set to true, checkups will sync upstram statuses
+    -- from checkups to Nginx upstream blocks.
+    checkup_shd_sync_enable = true,
+
+    -- Interval of syncing upstream status from checkups
+    -- to Nginx upstream blocks.
+    shd_config_timer_interval = 1,
+}
+
+
+_M.snapshot = {
+    -- Enable or disable heartbeats to servers
+    enable = false,
+
+    -- Cluster type, must be one of general, redis, mysql, http. 
+    typ = "http",
+
+    -- Limits the time during which a request can be responsed,
+    -- likewise nginx proxy_next_upstream_timeout.
+    try_timeout = 30,
+
+    --  Connect timeout to upstream servers.
+    timeout = 15,
+
+    -- Retry count. Default is the number of servers.
+    try = 100,
+    -- If set to true and all the servers in the cluster are failing
+    -- checkups will not mark the last failing server as unavailable(err)
+    -- instead, it will be marked as unstable(still available in next try)
+    protected = true,
+
+    -- cluster configure info
+   cluster = {
+        {   -- level 1
+            servers = {
+                { host = "10.182.26.25", port = 5555 },
+                { host = "10.182.26.17", port = 5555 },
+                { host = "10.182.26.16", port = 5555 },
+            }
+        }
+    }
+}
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/argutils.lua

@@ -0,0 +1,233 @@
+-- Copyright (C) 2015 Jingli Chen (Wine93), UPYUN Inc.
+
+local ngx_find     = ngx.re.find
+local escape_uri   = ngx.escape_uri
+
+local sort   = table.sort
+local insert = table.insert
+local concat = table.concat
+local gsub   = string.gsub
+
+local _M = {
+    _VERSION = "0.04"
+}
+
+
+local function split(s, p)
+    local res = {}
+    gsub(s, "([^" .. p .. "]+)", function(w)
+            if #w > 0 then insert(res, w) end
+    end)
+    return res
+end
+
+
+local function match(s, p, ci)
+    local opts = "jo"
+    if ci == true then opts = "ijo" end
+    return ngx_find(s, p, opts)
+end
+
+
+local function parse_arg(arg)
+    if type(arg) ~= "string" or #arg < 1 then
+        return nil, nil, "invalid argument"
+    end
+
+    local from = arg:find("=")
+    if from then
+        local key = arg:sub(1, from - 1)
+        local val = arg:sub(from + 1)
+        if #key > 0 then
+            return key, val
+        end
+    else  -- only key, like ?a
+        return arg, true
+    end
+end
+
+
+local function insert_arg(t, k, v)
+    if not t[k] then
+        t[k] = v
+        return
+    end
+
+    if type(t[k]) == "table" then
+        insert(t[k], v)
+    else
+        t[k] = { t[k], v }
+    end
+end
+
+
+function _M.LESS(a, b)
+    if a.key == b.key then
+        if tostring(a.val) ~= tostring(b.val) then
+            if type(a.val) == "boolean" then
+                return true
+            elseif type(b.val) ~= "boolean"
+                and tostring(a.val) < tostring(b.val) then
+                return true
+            end
+        end
+    end
+
+    return a.key < b.key
+end
+
+
+function _M.sort_args(args, opts)
+    if type(args) ~= "string" then
+        return nil, "non-string arguments"
+    end
+
+    if type(opts) ~= "table" then opts = {} end
+    local order = opts.order or _M.LESS
+
+    if type(order) ~= "function" then
+        return nil, "non-function order"
+    end
+
+    local sorted_args = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            insert(sorted_args, {
+                key = key, val = val
+            })
+        end
+    end
+
+    -- sort the aguments by order function
+    sort(sorted_args, order)
+
+    local res = {}
+    for _, arg in ipairs(sorted_args) do
+        local key = arg.key
+        local val = arg.val
+        if type(val) == "boolean" then
+            insert(res, key)
+        else
+            insert(res, key .. "=" .. val)
+        end
+    end
+
+    return concat(res, '&')
+end
+
+
+function _M.match_args(args, key_pattern, val_pattern, opts)
+    if type(args) ~= "string" then
+        return nil, "non-string argument"
+    end
+    if type(key_pattern) ~= "string" then
+        return nil, "non-string key regex expression"
+    end
+    if type(val_pattern) ~= "string" then
+        return nil, "non-string val regex expression"
+    end
+
+    -- ci == case insensitive
+    if type(opts) ~= "table" then opts = {} end
+    local key_ci = opts.key_ci
+    local val_ci = opts.val_ci
+    local match_bool_value = opts.match_bool_value
+
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            if type(val) == "boolean" then
+                if match_bool_value
+                and match(key, key_pattern, key_ci) then
+                    insert_arg(res, key, val)
+                end
+            elseif type(val) == "string" then
+                if match(key, key_pattern, key_ci)
+                and match(val, val_pattern, val_ci) then
+                    insert_arg(res, key, val)
+                end
+            end
+        end
+    end
+
+    return res
+end
+
+
+function _M.delete_args(args, key_pattern, val_pattern, opts)
+    if type(args) ~= "string" then
+        return nil, "non-string argument"
+    end
+    if type(key_pattern) ~= "string" then
+        return nil, "non-string key regex expression"
+    end
+    if type(val_pattern) ~= "string" then
+        return nil, "non-string val regex expression"
+    end
+
+    -- ci == case insensitive
+    if type(opts) ~= "table" then opts = {} end
+    local key_ci = opts.key_ci
+    local val_ci = opts.val_ci
+    local delete_bool_value = opts.delete_bool_value
+
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            if type(val) == "boolean" then
+                if not delete_bool_value
+                or not match(key, key_pattern, key_ci) then
+                    insert(res, key)
+                end
+            elseif type(val) == "string" then
+                if not match(key, key_pattern, key_ci)
+                or not match(val, val_pattern, val_ci) then
+                    insert(res, key .. "=" .. val)
+                end
+            end
+        end
+    end
+
+    return concat(res, '&')
+end
+
+
+function _M.escape_args(args)
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            key = escape_uri(key)
+            if type(val) == "boolean" then
+                insert(res, key)
+            elseif type(val) == "string" then
+                val = escape_uri(val)
+                insert(res, key .. "=" .. val)
+            end
+        end
+    end
+
+    return concat(res, '&')
+end
+
+
+function _M.split_args(args)
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        insert(res, {key, val})
+    end
+
+    return res
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups.lua

@@ -0,0 +1,5 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local api = require "resty.checkups.api"
+
+return api

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/api.lua

@@ -0,0 +1,319 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local cjson      = require "cjson.safe"
+local lrucache   = require "resty.lrucache.pureffi"
+local heartbeat  = require "resty.checkups.heartbeat"
+local dyconfig   = require "resty.checkups.dyconfig"
+local base       = require "resty.checkups.base"
+local try        = require "resty.checkups.try"
+
+
+local localtime  = ngx.localtime
+local mutex      = ngx.shared.mutex
+local state      = ngx.shared.state
+local shd_config = ngx.shared.config
+local log        = ngx.log
+local now        = ngx.now
+local ERR        = ngx.ERR
+local WARN       = ngx.WARN
+local INFO       = ngx.INFO
+local worker_id  = ngx.worker.id
+local get_phase  = ngx.get_phase
+local str_format = string.format
+local find       = string.find
+local type       = type
+local next       = next
+local pairs      = pairs
+local ipairs     = ipairs
+local pcall      = pcall
+
+
+local _M = {
+    _VERSION = "0.20",
+    STATUS_OK       = base.STATUS_OK,
+    STATUS_UNSTABLE = base.STATUS_UNSTABLE,
+    STATUS_ERR      = base.STATUS_ERR
+}
+
+
+function _M.feedback_status(skey, host, port, failed)
+    local ups = base.upstream.checkups[skey]
+    if not ups then
+        return nil, "unknown skey " .. skey
+    end
+
+    local srv
+    for level, cls in pairs(ups.cluster) do
+        for _, s in ipairs(cls.servers) do
+            if s.host == host and s.port == port then
+                srv = s
+                break
+            end
+        end
+    end
+
+    if not srv then
+        return nil, "unknown host:port" .. host .. ":" .. port
+    end
+
+    base.set_srv_status(skey, srv, failed)
+    return 1
+end
+
+
+function _M.ready_ok(skey, callback, opts, upstream)
+    opts = opts or {}
+    local ups = upstream or base.upstream.checkups[skey]
+    if not ups then
+        return nil, "unknown skey " .. skey
+    end
+
+    return try.try_cluster(skey, callback, opts, ups)
+end
+
+
+function _M.init(config)
+    if not config.global.checkup_shd_sync_enable then
+        return true
+    end
+
+    local skeys = {}
+    for skey, ups in pairs(config) do repeat
+        if type(ups) == "table" and type(ups.cluster) == "table" then
+            for level, cls in pairs(ups.cluster) do
+                base.extract_servers_from_upstream(skey, cls)
+            end
+
+            local key = dyconfig._gen_shd_key(skey)
+
+            local encode_status, dup_ups = pcall(cjson.encode, base.table_dup(ups))
+            if encode_status == false then break end
+
+            local ok, err = shd_config:set(key, dup_ups)
+            if not ok then
+                return nil, err
+            end
+        end
+        skeys[skey] = 1
+    until true end
+
+    local ok, err = shd_config:set(base.SHD_CONFIG_VERSION_KEY, 0)
+    if not ok then
+        return nil, err
+    end
+
+    local ok, err = shd_config:set(base.SKEYS_KEY, cjson.encode(skeys))
+    if not ok then
+        return nil, err
+    end
+
+    return true
+end
+
+
+function _M.prepare_checker(config)
+    base.upstream.start_time = localtime()
+    base.upstream.conf_hash = config.global.conf_hash
+    base.upstream.checkup_timer_interval = config.global.checkup_timer_interval or 5
+    base.upstream.checkup_timer_overtime = config.global.checkup_timer_overtime or 60
+    base.upstream.ups_status_sync_enable = config.global.ups_status_sync_enable
+    base.upstream.ups_status_timer_interval = config.global.ups_status_timer_interval or 5
+    base.upstream.checkup_shd_sync_enable = config.global.checkup_shd_sync_enable
+    base.upstream.shd_config_timer_interval = config.global.shd_config_timer_interval
+        or base.upstream.checkup_timer_interval
+    base.upstream.default_heartbeat_enable = config.global.default_heartbeat_enable
+
+    base.upstream.checkups = {}
+    local cluster_status = lrucache.new(config.global.cdn_lrucache_max_items or 1000)
+    local expired = config.global.cdn_srvs_status_expires or 300
+    base.init_cluster_status(cluster_status, expired)
+
+    for skey, ups in pairs(config) do
+        if type(ups) == "table" and type(ups.cluster) == "table" then
+            base.upstream.checkups[skey] = base.table_dup(ups)
+
+            for level, cls in pairs(base.upstream.checkups[skey].cluster) do
+                base.extract_servers_from_upstream(skey, cls)
+            end
+        end
+    end
+
+    if base.upstream.checkup_shd_sync_enable then
+        base.upstream.shd_config_version = 0
+    end
+
+    base.upstream.initialized = true
+end
+
+
+function _M.get_status()
+    local all_status = {}
+    for skey in pairs(base.upstream.checkups) do
+        all_status["cls:" .. skey] = base.get_upstream_status(skey)
+    end
+    local last_check_time = state:get(base.CHECKUP_LAST_CHECK_TIME_KEY) or cjson.null
+    all_status.last_check_time = last_check_time
+    all_status.checkup_timer_alive = state:get(base.CHECKUP_TIMER_ALIVE_KEY) or false
+    all_status.start_time = base.upstream.start_time
+    all_status.conf_hash = base.upstream.conf_hash or cjson.null
+    all_status.shd_config_version = base.upstream.shd_config_version or cjson.null
+
+    all_status.config_timer = dyconfig.get_timer_key_status()
+
+    return all_status
+end
+
+
+function _M.get_ups_timeout(skey)
+    if not skey then
+        return
+    end
+
+    local ups = base.upstream.checkups[skey]
+    if not ups then
+        return
+    end
+
+    local timeout = ups.timeout or 5
+    return timeout, ups.send_timeout or timeout, ups.read_timeout or timeout
+end
+
+
+function _M.create_checker()
+    local phase = get_phase()
+    if phase ~= "init_worker" then
+        error("create_checker must be called in init_worker phase")
+    end
+
+    if not base.upstream.initialized then
+        log(ERR, "create checker failed, call prepare_checker in init_by_lua")
+        return
+    end
+
+    -- shd config syncer enabled
+    if base.upstream.shd_config_version then
+        dyconfig.create_shd_config_syncer()
+    end
+
+    if base.upstream.ups_status_sync_enable and not base.ups_status_timer_created then
+        local ok, err = ngx.timer.at(0, base.ups_status_checker)
+        if not ok then
+            log(WARN, "failed to create ups_status_checker: ", err)
+            return
+        end
+        base.ups_status_timer_created = true
+    end
+
+    if not worker_id then
+        log(ERR, "ngx_http_lua_module version too low, no heartbeat timer will be created")
+        return
+    elseif worker_id() ~= 0 then
+        return
+    end
+
+    -- only worker 0 will create heartbeat timer
+    local ok, err = ngx.timer.at(0, heartbeat.active_checkup)
+    if not ok then
+        log(WARN, "failed to create timer: ", err)
+        return
+    end
+
+    local ckey = base.CHECKUP_TIMER_KEY
+    local overtime = base.upstream.checkup_timer_overtime
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+end
+
+
+function _M.select_peer(skey, ups, opts)
+    return _M.ready_ok(skey, function(host, port)
+        return { host=host, port=port }
+    end, opts, ups)
+end
+
+
+local function gen_upstream(skey, upstream)
+    local ups = upstream
+    if upstream.cluster then
+        -- all upstream
+        if type(upstream.cluster) ~= "table" then
+            return nil, "cluster invalid"
+        end
+    else
+        -- only servers
+        local dyupstream, err = dyconfig.do_get_upstream(skey)
+        if err then
+            return nil, err
+        end
+
+        dyupstream = dyupstream or {}
+        dyupstream.cluster = upstream
+        ups = dyupstream
+    end
+
+    -- check servers
+    local ok
+    for level, cls in pairs(ups.cluster) do
+        if not cls or not next(cls) then
+            return nil, "can not update empty level"
+        end
+
+        local servers = cls.servers
+        if not servers or not next(servers) then
+            return nil, "can not update empty servers"
+        end
+
+        for _, srv in ipairs(servers) do
+            local ok, err = dyconfig.check_update_server_args(skey, level, srv)
+            if not ok then
+                return nil, err
+            end
+        end
+    end
+
+    return ups
+end
+
+
+function _M.update_upstream(skey, upstream)
+    if not upstream or not next(upstream) then
+        return false, "can not set empty upstream"
+    end
+
+    local lock, err = base.get_lock(base.SKEYS_KEY)
+    if not lock then
+        log(WARN, "failed to acquire the lock: ", err)
+        return false, err
+    end
+
+    local ups, err = gen_upstream(skey, upstream)
+    local ok = false
+    if not err then
+        ok, err = dyconfig.do_update_upstream(skey, ups)
+    end
+
+    base.release_lock(lock)
+
+    return ok, err
+end
+
+
+function _M.delete_upstream(skey)
+    local lock, ok, err
+    lock, err = base.get_lock(base.SKEYS_KEY)
+    if not lock then
+        log(WARN, "failed to acquire the lock: ", err)
+        return false, err
+    end
+
+    ok, err = dyconfig.do_delete_upstream(skey)
+
+    base.release_lock(lock)
+
+    return ok, err
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/base.lua

@@ -0,0 +1,367 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local cjson      = require "cjson.safe"
+local lock       = require "resty.lock"
+
+local str_format = string.format
+local str_sub    = string.sub
+local str_find   = string.find
+local str_match  = string.match
+local tab_insert = table.insert
+local unpack     = unpack
+local tostring   = tostring
+local ipairs     = ipairs
+local pairs      = pairs
+local type       = type
+
+local log        = ngx.log
+local ERR        = ngx.ERR
+local WARN       = ngx.WARN
+local state      = ngx.shared.state
+local now        = ngx.now
+
+
+local _M = {
+    _VERSION = "0.20",
+    STATUS_OK = 0,
+    STATUS_UNSTABLE = 1,
+    STATUS_ERR = 2
+}
+
+local ngx_upstream
+
+local CHECKUP_TIMER_KEY = "checkups:timer"
+_M.CHECKUP_TIMER_KEY = CHECKUP_TIMER_KEY
+local CHECKUP_LAST_CHECK_TIME_KEY = "checkups:last_check_time"
+_M.CHECKUP_LAST_CHECK_TIME_KEY = CHECKUP_LAST_CHECK_TIME_KEY
+local CHECKUP_TIMER_ALIVE_KEY = "checkups:timer_alive"
+_M.CHECKUP_TIMER_ALIVE_KEY = CHECKUP_TIMER_ALIVE_KEY
+local PEER_STATUS_PREFIX = "checkups:peer_status:"
+_M.PEER_STATUS_PREFIX = PEER_STATUS_PREFIX
+local SHD_CONFIG_VERSION_KEY = "config_version"
+_M.SHD_CONFIG_VERSION_KEY = SHD_CONFIG_VERSION_KEY
+local SKEYS_KEY = "checkups:skeys"
+_M.SKEYS_KEY = SKEYS_KEY
+local SHD_CONFIG_PREFIX = "shd_config"
+_M.SHD_CONFIG_PREFIX = SHD_CONFIG_PREFIX
+
+
+local upstream = {}
+_M.upstream = upstream
+local peer_id_dict = {}
+
+local expired
+local cluster_status
+
+local ups_status_timer_created
+_M.ups_status_timer_created = ups_status_timer_created
+
+
+_M.is_tab = function(t) return type(t) == "table" end
+_M.is_str = function(t) return type(t) == "string" end
+_M.is_num = function(t) return type(t) == "number" end
+_M.is_nul = function(t) return t == nil or t == ngx.null end
+
+
+local function _gen_key(skey, srv)
+    return str_format("%s:%s:%d", skey, srv.host, srv.port)
+end
+_M._gen_key = _gen_key
+
+
+local function extract_srv_host_port(name)
+    local from, to = str_find(name, ":")
+    if from then
+        local host = str_sub(name, 1, from - 1)
+        local port = str_sub(name, to + 1)
+        host = str_match(host, "^%d+%.%d+%.%d+%.%d+$")
+        port = str_match(port, "^%d+$")
+        if host and port then
+            return host, port
+        end
+    else
+        local host = str_match(name, "^%d+%.%d+%.%d+%.%d+$")
+        if host then
+            return host, 80
+        end
+    end
+end
+
+
+function _M.init_cluster_status(_cluster_status, _expired)
+    expired = _expired
+    cluster_status = _cluster_status
+end
+
+
+function _M.get_srv_status(skey, srv, id)
+    local server_status = cluster_status:get(skey)
+    if not server_status then
+        return _M.STATUS_OK
+    end
+
+    local srv_key = str_format("%s:%d:%s:%s", srv.host, srv.port, srv.isp or "", id)
+    local srv_status = server_status[srv_key]
+    local fail_timeout = srv.fail_timeout or 10
+
+    if srv_status and srv_status.lastmodify + fail_timeout > now() then
+        return srv_status.status
+    end
+
+    return _M.STATUS_OK
+end
+
+
+function _M.set_srv_status(skey, srv, id, failed, _ups)
+    local server_status = cluster_status:get(skey)
+    if not server_status then
+        server_status = {}
+        cluster_status:set(skey, server_status, expired)
+    end
+
+    -- The default max_fails is 0, which differs from nginx upstream module(1).
+    local max_fails = srv.max_fails or 0
+    local fail_timeout = srv.fail_timeout or 10
+    if max_fails == 0 then  -- disables the accounting of attempts
+        return
+    end
+
+    local time_now = now()
+    local srv_key = str_format("%s:%d:%s:%s", srv.host, srv.port, srv.isp or "", id)
+    local srv_status = server_status[srv_key]
+    if not srv_status then  -- first set
+        srv_status = {
+            status = _M.STATUS_OK,
+            failed_count = 0,
+            lastmodify = time_now
+        }
+        server_status[srv_key] = srv_status
+    elseif srv_status.lastmodify + fail_timeout < time_now then -- srv_status expired
+        srv_status.status = _M.STATUS_OK
+        srv_status.failed_count = 0
+        srv_status.lastmodify = time_now
+    end
+
+    if failed then
+        srv_status.failed_count = srv_status.failed_count + 1
+
+        if srv_status.failed_count >= max_fails then
+            local ups = _ups or upstream.checkups[skey]
+            for level, cls in pairs(ups.cluster) do
+                for _, s in ipairs(cls.servers) do
+                    local k = str_format("%s:%d:%s:%s", s.host, s.port, s.isp or "", id)
+                    local st = server_status[k]
+                    -- not the last ok server
+                    if (not st or st.status == _M.STATUS_OK) and k ~= srv_key then
+                        srv_status.status = _M.STATUS_ERR
+                        srv_status.lastmodify = time_now
+                        return
+                    end
+                end
+            end
+        end
+    end
+end
+
+
+function _M.get_lock(key, timeout)
+    local lock = lock:new("locks", {timeout=timeout})
+    local elapsed, err = lock:lock(key)
+    if not elapsed then
+        log(WARN, "failed to acquire the lock: ", key, ", ", err)
+        return nil, err
+    end
+
+    return lock
+end
+
+
+function _M.release_lock(lock)
+    local ok, err = lock:unlock()
+    if not ok then
+        log(WARN, "failed to unlock: ", err)
+    end
+end
+
+
+function _M.get_peer_status(skey, srv)
+    local peer_key = PEER_STATUS_PREFIX .. _gen_key(skey, srv)
+    local peer_status = state:get(peer_key)
+    return not _M.is_nul(peer_status) and cjson.decode(peer_status) or nil
+end
+
+
+function _M.get_upstream_status(skey)
+    local ups = upstream.checkups[skey]
+    if not ups then
+        return
+    end
+
+    local ups_status = {}
+
+    for level, cls in pairs(ups.cluster) do
+        local servers = cls.servers
+        ups_status[level] = {}
+        if servers and type(servers) == "table" and #servers > 0 then
+            for _, srv in ipairs(servers) do
+                local peer_status = _M.get_peer_status(skey, srv) or {}
+                peer_status.server = _gen_key(skey, srv)
+                peer_status["weight"] = srv.weight
+                peer_status["max_fails"] = srv.max_fails
+                peer_status["fail_timeout"] = srv.fail_timeout
+                if ups.enable == false or (ups.enable == nil and
+                   upstream.default_heartbeat_enable == false) then
+                    peer_status.status = "unchecked"
+                else
+                    if not peer_status.status or
+                       peer_status.status == _M.STATUS_OK then
+                        peer_status.status = "ok"
+                    elseif peer_status.status == _M.STATUS_ERR then
+                        peer_status.status = "err"
+                    else
+                        peer_status.status = "unstable"
+                    end
+                end
+                tab_insert(ups_status[level], peer_status)
+            end
+        end
+    end
+
+    return ups_status
+end
+
+
+function _M.extract_servers_from_upstream(skey, cls)
+    local up_key = cls.upstream
+    if not up_key then
+        return
+    end
+
+    cls.servers = cls.servers or {}
+
+    if not ngx_upstream then
+        local ok
+        ok, ngx_upstream = pcall(require, "ngx.upstream")
+        if not ok then
+            log(ERR, "ngx_upstream_lua module required")
+            return
+        end
+    end
+
+    local ups_backup = cls.upstream_only_backup
+    local srvs_getter = ngx_upstream.get_primary_peers
+    if ups_backup then
+        srvs_getter = ngx_upstream.get_backup_peers
+    end
+    local srvs, err = srvs_getter(up_key)
+    if not srvs and err then
+        log(ERR, "failed to get servers in upstream, key: ", up_key, " err:",  err)
+        return
+    end
+
+    for _, srv in ipairs(srvs) do
+        local host, port = extract_srv_host_port(srv.name)
+        if not host then
+            log(ERR, "invalid server name: ", srv.name)
+            return
+        end
+        peer_id_dict[_gen_key(skey, { host = host, port = port })] = {
+            id = srv.id, backup = ups_backup and true or false}
+        tab_insert(cls.servers, {
+            host = host,
+            port = port,
+            weight = srv.weight,
+            max_fails = srv.max_fails,
+            fail_timeout = srv.fail_timeout,
+        })
+    end
+end
+
+
+function _M.table_dup(ori_tab)
+    if type(ori_tab) ~= "table" then
+        return ori_tab
+    end
+    local new_tab = {}
+    for k, v in pairs(ori_tab) do
+        if type(v) == "table" then
+            new_tab[k] = _M.table_dup(v)
+        else
+            new_tab[k] = v
+        end
+    end
+    return new_tab
+end
+
+
+function _M.ups_status_checker(premature)
+    if premature then
+        return
+    end
+
+    if not ngx_upstream then
+        local ok
+        ok, ngx_upstream = pcall(require, "ngx.upstream")
+        if not ok then
+            log(ERR, "ngx_upstream_lua module required")
+            return
+        end
+    end
+
+    local ups_status = {}
+    local names = ngx_upstream.get_upstreams()
+    -- get current upstream down status
+    for _, name in ipairs(names) do
+        local srvs = ngx_upstream.get_primary_peers(name)
+        for _, srv in ipairs(srvs) do
+            ups_status[srv.name] = srv.down and _M.STATUS_ERR or _M.STATUS_OK
+        end
+
+        srvs = ngx_upstream.get_backup_peers(name)
+        for _, srv in ipairs(srvs) do
+            ups_status[srv.name] = srv.down and _M.STATUS_ERR or _M.STATUS_OK
+        end
+    end
+
+    for skey, ups in pairs(upstream.checkups) do
+        for level, cls in pairs(ups.cluster) do
+            if not cls.upstream then
+                break
+            end
+
+            for _, srv in pairs(cls.servers) do
+                local peer_key = _gen_key(skey, srv)
+                local status_key = PEER_STATUS_PREFIX .. peer_key
+
+                local peer_status, err = state:get(status_key)
+                if peer_status then
+                    local st = cjson.decode(peer_status)
+                    local up_st = ups_status[srv.host .. ':' .. srv.port]
+                    local unstable = st.status == _M.STATUS_UNSTABLE
+                    if (unstable and up_st == _M.STATUS_ERR) or
+                        (not unstable and up_st and st.status ~= up_st) then
+                        local up_id = peer_id_dict[peer_key]
+                        local down = up_st == _M.STATUS_OK
+                        local ok, err = ngx_upstream.set_peer_down(
+                            cls.upstream, up_id.backup, up_id.id, down)
+                        if not ok then
+                            log(ERR, "failed to set peer down", err)
+                        end
+                    end
+                elseif err then
+                    log(WARN, "get peer status error ", status_key, " ", err)
+                end
+            end
+        end
+    end
+
+    local interval = upstream.ups_status_timer_interval
+    local ok, err = ngx.timer.at(interval, _M.ups_status_checker)
+    if not ok then
+        ups_status_timer_created = false
+        log(WARN, "failed to create ups_status_checker: ", err)
+    end
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/consistent_hash.lua

@@ -0,0 +1,99 @@
+-- Copyright (C) 2014-2016, UPYUN Inc.
+
+local floor      = math.floor
+local str_byte   = string.byte
+local tab_sort   = table.sort
+local tab_insert = table.insert
+
+local _M = { _VERSION = "0.11" }
+
+local MOD       = 2 ^ 32
+local REPLICAS  = 20
+local LUCKY_NUM = 13
+
+
+local function hash_string(str)
+    local key = 0
+    for i = 1, #str do
+        key = (key * 31 + str_byte(str, i)) % MOD
+    end
+
+    return key
+end
+
+
+local function init_consistent_hash_state(servers)
+    local weight_sum = 0
+    for _, srv in ipairs(servers) do
+        weight_sum = weight_sum + (srv.weight or 1)
+    end
+
+    local circle, members = {}, 0
+    for index, srv in ipairs(servers) do
+        local key = ("%s:%s"):format(srv.host, srv.port)
+        local base_hash = hash_string(key)
+        for c = 1, REPLICAS * weight_sum do
+            -- TODO: more balance hash
+            local hash = (base_hash * c * LUCKY_NUM) % MOD
+            tab_insert(circle, { hash, index })
+        end
+        members = members + 1
+    end
+
+    tab_sort(circle, function(a, b) return a[1] < b[1] end)
+
+    return { circle = circle, members = members }
+end
+
+
+local function binary_search(circle, key)
+    local size = #circle
+    local st, ed, mid = 1, size
+    while st <= ed do
+        mid = floor((st + ed) / 2)
+        if circle[mid][1] < key then
+            st = mid + 1
+        else
+            ed = mid - 1
+        end
+    end
+
+    return st == size + 1 and 1 or st
+end
+
+
+function _M.next_consistent_hash_server(servers, peer_cb, hash_key)
+    local is_tab = require "resty.checkups.base".is_tab
+    servers.chash = is_tab(servers.chash) and servers.chash
+                    or init_consistent_hash_state(servers)
+
+    local chash = servers.chash
+    if chash.members == 1 then
+        if peer_cb(1, servers[1]) then
+            return servers[1]
+        end
+
+        return nil, "consistent hash: no servers available"
+    end
+
+    local circle = chash.circle
+    local st = binary_search(circle, hash_string(hash_key))
+    local size = #circle
+    local ed = st + size - 1
+    for i = st, ed do  -- TODO: algorithm O(n)
+        local idx = circle[(i - 1) % size + 1][2]
+        if peer_cb(idx, servers[idx]) then
+            return servers[idx]
+        end
+    end
+
+    return nil, "consistent hash: no servers available"
+end
+
+
+function _M.free_consitent_hash_server(srv, failed)
+    return
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/dyconfig.lua

@@ -0,0 +1,290 @@
+local cjson         = require "cjson.safe"
+
+local base          = require "resty.checkups.base"
+
+local worker_id     = ngx.worker.id
+local worker_count  = ngx.worker.count
+local update_time   = ngx.update_time
+local mutex         = ngx.shared.mutex
+local state         = ngx.shared.state
+local shd_config    = ngx.shared.config
+local log           = ngx.log
+local ERR           = ngx.ERR
+local WARN          = ngx.WARN
+local INFO          = ngx.INFO
+
+local str_format    = string.format
+local type          = type
+
+local _M = {
+    _VERSION = "0.11",
+    STATUS_OK = base.STATUS_OK, STATUS_UNSTABLE = base.STATUS_UNSTABLE, STATUS_ERR = base.STATUS_ERR
+}
+
+local function _gen_shd_key(skey)
+    return str_format("%s:%s", base.SHD_CONFIG_PREFIX, skey)
+end
+_M._gen_shd_key = _gen_shd_key
+
+
+local function shd_config_syncer(premature)
+    local ckey = base.CHECKUP_TIMER_KEY .. ":shd_config:" .. worker_id()
+    update_time()
+
+    if premature then
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+
+    local interval = base.upstream.shd_config_timer_interval
+
+    local overtime = base.upstream.checkup_timer_overtime
+
+    local lock, err = base.get_lock(base.SKEYS_KEY)
+    if not lock then
+        log(WARN, "upstream updating, failed to acquire the lock: ", base.SKEYS_KEY, ", ", err)
+        local ok, err = ngx.timer.at(interval, shd_config_syncer)
+        if not ok then
+            log(ERR, "failed to create timer: ", err)
+            local ok, err = mutex:set(ckey, nil)
+            if not ok then
+                log(ERR, "failed to update shm: ", err)
+            end
+        else
+            local ok, err = mutex:set(ckey, 1, overtime)
+            if not ok then
+                log(ERR, "failed to update shm: ", err)
+            end
+        end
+        return
+    end
+
+    local config_version, err = shd_config:get(base.SHD_CONFIG_VERSION_KEY)
+
+    if config_version and config_version ~= base.upstream.shd_config_version then
+        local skeys = shd_config:get(base.SKEYS_KEY)
+        if skeys then
+            skeys = cjson.decode(skeys)
+
+            -- delete skey from upstream
+            for skey, _ in pairs(base.upstream.checkups) do
+                if not skeys[skey] then
+                    base.upstream.checkups[skey] = nil
+                end
+            end
+
+            local success = true
+            for skey, _ in pairs(skeys) do
+                local shd_servers, err = shd_config:get(_gen_shd_key(skey))
+                log(INFO, "get ", skey, " from shm: ", shd_servers, " err: ", err)
+                if shd_servers then
+                    shd_servers = cjson.decode(shd_servers)
+                    base.upstream.checkups[skey] = base.table_dup(shd_servers)
+                elseif err then
+                    success = false
+                    log(WARN, "failed to get from shm: ", err)
+                end
+            end
+
+            if success then
+                base.upstream.shd_config_version = config_version
+            end
+        end
+    elseif err then
+        log(WARN, "failed to get config version from shm")
+    end
+
+    base.release_lock(lock)
+
+
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+
+    local ok, err = ngx.timer.at(interval, shd_config_syncer)
+    if not ok then
+        log(ERR, "failed to create timer: ", err)
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+end
+
+_M.shd_config_syncer = shd_config_syncer
+
+
+function _M.check_update_server_args(skey, level, server)
+    if type(skey) ~= "string" then
+        return false, "skey must be a string"
+    end
+    if type(level) ~= "number" and type(level) ~= "string" then
+        return false, "level must be string or number"
+    end
+    if type(server) ~= "table" then
+        return false, "server must be a table"
+    end
+    if not server.host or not server.port then
+        return false, "no server.host nor server.port found"
+    end
+
+    return true
+end
+
+
+function _M.do_get_upstream(skey)
+    local skeys = shd_config:get(base.SKEYS_KEY)
+    if not skeys then
+        return nil, "no skeys found from shm"
+    end
+
+    local key = _gen_shd_key(skey)
+    local shd_servers, err = shd_config:get(key)
+    if shd_servers then
+        shd_servers = cjson.decode(shd_servers)
+        if type(shd_servers) ~= "table" then
+            return nil
+        end
+
+        return shd_servers
+    elseif err then
+        log(WARN, "failed to get from shm: ", err)
+        return nil, err
+    else
+        log(WARN, "upstream " .. skey .. " not found")
+        return nil
+    end
+end
+
+
+function _M.do_update_upstream(skey, upstream)
+    local skeys = shd_config:get(base.SKEYS_KEY)
+    if not skeys then
+        return false, "no skeys found from shm"
+    end
+
+    skeys = cjson.decode(skeys)
+
+    local new_ver, ok, err
+
+    new_ver, err = shd_config:incr(base.SHD_CONFIG_VERSION_KEY, 1)
+
+    if err then
+        log(WARN, "failed to set new version to shm")
+        return false, err
+    end
+
+    local key = _gen_shd_key(skey)
+    ok, err = shd_config:set(key, cjson.encode(upstream))
+
+    if err then
+        log(WARN, "failed to set new upstream to shm")
+        return false, err
+    end
+
+    -- new skey
+    if not skeys[skey] then
+        skeys[skey] = 1
+        local _, err = shd_config:set(base.SKEYS_KEY, cjson.encode(skeys))
+        if err then
+            log(WARN, "failed to set new skeys to shm")
+            return false, err
+        end
+        log(INFO, "add new skey to upstreams, ", skey)
+    end
+
+    return true
+end
+
+
+function _M.do_delete_upstream(skey)
+    local skeys = shd_config:get(base.SKEYS_KEY)
+    if skeys then
+        skeys = cjson.decode(skeys)
+    else
+        return false, "upstream " .. skey .. " not found"
+    end
+
+    local key = _gen_shd_key(skey)
+    local shd_servers, err = shd_config:get(key)
+    if shd_servers then
+        local new_ver, ok, err
+        new_ver, err = shd_config:incr(base.SHD_CONFIG_VERSION_KEY, 1)
+        if err then
+            log(WARN, "failed to set new version to shm")
+            return false, err
+        end
+
+        ok, err = shd_config:delete(key)
+        if err then
+            log(WARN, "failed to delete servers in shm")
+            return false, err
+        end
+
+        skeys[skey] = nil
+
+        local _, err = shd_config:set(base.SKEYS_KEY, cjson.encode(skeys))
+        if err then
+            log(WARN, "failed to set new skeys to shm")
+            return false, err
+        end
+
+        log(INFO, "delete skey from upstreams, ", skey)
+
+    elseif err then
+        return false, err
+    else
+        return false, "upstream " .. skey .. " not found"
+    end
+
+    return true
+end
+
+
+function _M.create_shd_config_syncer()
+    local ok, err = ngx.timer.at(0, shd_config_syncer)
+    if not ok then
+        log(ERR, "failed to create shd_config timer: ", err)
+        return
+    end
+
+    local overtime = base.upstream.checkup_timer_overtime
+    local ckey = base.CHECKUP_TIMER_KEY .. ":shd_config:" .. worker_id()
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+end
+
+
+function _M.get_timer_key_status()
+    if not worker_count then
+        log(WARN, "can not get worker count, please upgrade lua-nginx-module to 0.9.20 or higher")
+        return
+    end
+
+    local timer_status = {}
+    local count = worker_count()
+    for i=0, count-1 do
+        local key = "worker-" .. i
+        local ckey = base.CHECKUP_TIMER_KEY .. ":shd_config:" .. i
+        local val, err = mutex:get(ckey)
+        if err then
+            timer_status[key] = err
+        elseif val then
+            timer_status[key] = "alive"
+        else
+            timer_status[key] = "dead"
+        end
+    end
+
+    return timer_status
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/heartbeat.lua

@@ -0,0 +1,479 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local cjson         = require "cjson.safe"
+
+local base          = require "resty.checkups.base"
+
+local str_sub       = string.sub
+local lower         = string.lower
+local tab_sort      = table.sort
+local tab_concat    = table.concat
+local tab_insert    = table.insert
+
+local re_gmatch     = ngx.re.gmatch
+local re_find       = ngx.re.find
+local log           = ngx.log
+local localtime     = ngx.localtime
+local ERR           = ngx.ERR
+local WARN          = ngx.WARN
+local now           = ngx.now
+local tcp           = ngx.socket.tcp
+local update_time   = ngx.update_time
+local mutex         = ngx.shared.mutex
+local state         = ngx.shared.state
+
+
+local _M = {
+    _VERSION = "0.11",
+    STATUS_OK = base.STATUS_OK, STATUS_UNSTABLE = base.STATUS_UNSTABLE, STATUS_ERR = base.STATUS_ERR
+}
+
+local resty_redis, resty_mysql
+
+
+local function update_peer_status(srv, sensibility)
+    local peer_key = srv.peer_key
+    local status_key = base.PEER_STATUS_PREFIX .. peer_key
+    local status_str, err = state:get(status_key)
+
+    if err then
+        log(ERR, "get old status ", status_key, " ", err)
+        return
+    end
+
+    local old_status, err
+    if status_str then
+        old_status, err = cjson.decode(status_str)
+        if err then
+            log(WARN, "decode old status error: ", err)
+        end
+    end
+
+    if not old_status then
+        old_status = {
+            status = _M.STATUS_OK,
+            fail_num = 0,
+            lastmodified = localtime(),
+        }
+    end
+
+    local status = srv.status
+    if status == _M.STATUS_OK then
+        if old_status.status ~= _M.STATUS_OK then
+            old_status.lastmodified = localtime()
+            old_status.status = _M.STATUS_OK
+        end
+        old_status.fail_num = 0
+    else  -- status == _M.STATUS_ERR or _M.STATUS_UNSTABLE
+        old_status.fail_num = old_status.fail_num + 1
+
+        if old_status.status ~= status and
+            old_status.fail_num >= sensibility then
+            old_status.status = status
+            old_status.lastmodified = localtime()
+        end
+    end
+
+    for k, v in pairs(srv.statuses) do
+        old_status[k] = v
+    end
+
+    local ok, err = state:set(status_key, cjson.encode(old_status))
+    if not ok then
+        log(ERR, "failed to set new status ", err)
+    end
+end
+
+
+local function update_upstream_status(ups_status, sensibility)
+    if not ups_status then
+        return
+    end
+
+    for _, srv in ipairs(ups_status) do
+        update_peer_status(srv, sensibility)
+    end
+end
+
+
+local heartbeat = {
+    general = function (host, port, ups)
+        local id = host .. ':' .. port
+
+        local sock = tcp()
+        sock:settimeout(ups.timeout * 1000)
+        local ok, err = sock:connect(host, port)
+        if not ok then
+            log(ERR, "failed to connect: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        sock:setkeepalive()
+
+        return _M.STATUS_OK
+    end,
+
+    redis = function (host, port, ups)
+        local id = host .. ':' .. port
+
+        if not resty_redis then
+            local ok
+            ok, resty_redis = pcall(require, "resty.redis")
+            if not ok then
+                log(ERR, "failed to require resty.redis")
+                return _M.STATUS_ERR, "failed to require resty.redis"
+            end
+        end
+
+        local red, err = resty_redis:new()
+        if not red then
+            log(WARN, "failed to new redis: ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        red:set_timeout(ups.timeout * 1000)
+
+        local redis_err = { status = _M.STATUS_ERR, replication = cjson.null }
+        local ok, err = red:connect(host, port)
+        if not ok then
+            log(ERR, "failed to connect redis: ", id, ", ", err)
+            return redis_err, err
+        end
+
+        if ups.password then
+            local ok, err = red:auth(ups.password)
+            if err then
+                log(WARN, "failed to auth to redis:", err)
+            end
+        end
+
+        local res, err = red:ping()
+        if not res then
+            log(ERR, "failed to ping redis: ", id, ", ", err)
+            return redis_err, err
+        end
+
+        local replication = {}
+        local statuses = {
+            status = _M.STATUS_OK ,
+            replication = replication
+        }
+
+        local res, got_all_info = {}, false
+
+        local info, err = red:info("replication")
+        if not info then
+            info, err = red:info()
+            if not info then
+                replication.err = err
+                return statuses
+            end
+
+            got_all_info = true
+        end
+
+        tab_insert(res, info)
+
+        if not got_all_info then
+            local info, err = red:info("server")
+            if info then
+                tab_insert(res, info)
+            end
+        end
+
+        res = tab_concat(res)
+
+        red:set_keepalive(10000, 100)
+
+        local iter, err = re_gmatch(res, [[([a-zA-Z_]+):(.+?)\r\n]], "jo")
+        if not iter then
+            replication.err = err
+            return statuses
+        end
+
+        local replication_field = {
+            role                           = true,
+            master_host                    = true,
+            master_port                    = true,
+            master_link_status             = true,
+            master_link_down_since_seconds = true,
+            master_last_io_seconds_ago     = true,
+        }
+
+        local other_field = {
+            redis_version = true,
+        }
+
+        while true do
+            local m, err = iter()
+            if err then
+                replication.err = err
+                return statuses
+            end
+
+            if not m then
+                break
+            end
+
+            if replication_field[lower(m[1])] then
+                replication[m[1]] = m[2]
+            end
+
+            if other_field[lower(m[1])] then
+                statuses[m[1]] = m[2]
+            end
+        end
+
+        if replication.master_link_status == "down" then
+            statuses.status = _M.STATUS_UNSTABLE
+            statuses.msg = "master link status: down"
+        end
+
+        return statuses
+    end,
+
+    mysql = function (host, port, ups)
+        local id = host .. ':' .. port
+
+        if not resty_mysql then
+            local ok
+            ok, resty_mysql = pcall(require, "resty.mysql")
+            if not ok then
+                log(ERR, "failed to require resty.mysql")
+                return _M.STATUS_ERR, "failed to require resty.mysql"
+            end
+        end
+
+        local db, err = resty_mysql:new()
+        if not db then
+            log(WARN, "failed to new mysql: ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        db:set_timeout(ups.timeout * 1000)
+
+        local ok, err, errno, sqlstate = db:connect{
+            host = host,
+            port = port,
+            database = ups.name,
+            user = ups.user,
+            password = ups.pass,
+            max_packet_size = 1024*1024
+        }
+
+        if not ok then
+            log(ERR, "failed to connect: ", id, ", ", err, ": ", errno, " ", sqlstate)
+            return _M.STATUS_ERR, err
+        end
+
+        db:set_keepalive(10000, 100)
+
+        return _M.STATUS_OK
+    end,
+
+    http = function(host, port, ups)
+        local id = host .. ':' .. port
+
+        local sock, err = tcp()
+        if not sock then
+            log(WARN, "failed to create sock: ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        sock:settimeout(ups.timeout * 1000)
+        local ok, err = sock:connect(host, port)
+        if not ok then
+            log(ERR, "failed to connect: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        local opts = ups.http_opts or {}
+
+        local req = opts.query
+        if not req then
+            sock:setkeepalive()
+            return _M.STATUS_OK
+        end
+
+        local bytes, err = sock:send(req)
+        if not bytes then
+            log(ERR, "failed to send request to: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        local readline = sock:receiveuntil("\r\n")
+        local status_line, err = readline()
+        if not status_line then
+            log(ERR, "failed to receive status line from: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        local statuses = opts.statuses
+        if statuses then
+            local from, to, err = re_find(status_line,
+                [[^HTTP/\d+\.\d+\s+(\d+)]], "joi", nil, 1)
+            if not from then
+                log(ERR, "bad status line from: ", id, ", ", err)
+                return _M.STATUS_ERR, err
+            end
+
+            local status = str_sub(status_line, from, to)
+            if statuses[status] == false then
+                return _M.STATUS_ERR, "bad status code"
+            end
+        end
+
+        sock:setkeepalive()
+
+        return _M.STATUS_OK
+    end,
+}
+
+
+local function cluster_heartbeat(skey)
+    local ups = base.upstream.checkups[skey]
+    if ups.enable == false or (ups.enable == nil and
+        base.upstream.default_heartbeat_enable == false) then
+        return
+    end
+
+    local ups_typ = ups.typ or "general"
+    local ups_heartbeat = ups.heartbeat
+    local ups_sensi = ups.sensibility or 1
+    local ups_protected = true
+    if ups.protected == false then
+        ups_protected = false
+    end
+
+    ups.timeout = ups.timeout or 5
+
+    local server_count = 0
+    for level, cls in pairs(ups.cluster) do
+        if cls.servers and #cls.servers > 0 then
+            server_count = server_count + #cls.servers
+        end
+    end
+
+    local error_count = 0
+    local unstable_count = 0
+    local srv_available = false
+    local ups_status = {}
+    for level, cls in pairs(ups.cluster) do
+        for _, srv in ipairs(cls.servers) do
+            local peer_key = base._gen_key(skey, srv)
+            local cb_heartbeat = ups_heartbeat or heartbeat[ups_typ] or
+                heartbeat["general"]
+            local statuses, err = cb_heartbeat(srv.host, srv.port, ups)
+
+            local status
+            if type(statuses) == "table" then
+                status = statuses.status
+                statuses.status = nil
+            else
+                status = statuses
+                statuses = {}
+            end
+
+            if not statuses.msg then
+                statuses.msg = err or cjson.null
+            end
+
+            local srv_status = {
+                peer_key = peer_key ,
+                status   = status   ,
+                statuses = statuses ,
+            }
+
+            if status == _M.STATUS_OK then
+                update_peer_status(srv_status, ups_sensi)
+                srv_status.updated = true
+                srv_available = true
+                if next(ups_status) then
+                    for _, v in ipairs(ups_status) do
+                        if v.status == _M.STATUS_UNSTABLE then
+                            v.status = _M.STATUS_ERR
+                        end
+                        update_peer_status(v, ups_sensi)
+                    end
+                    ups_status = {}
+                end
+            end
+
+            if status == _M.STATUS_ERR then
+                error_count = error_count + 1
+                if srv_available then
+                    update_peer_status(srv_status, ups_sensi)
+                    srv_status.updated = true
+                end
+            end
+
+            if status == _M.STATUS_UNSTABLE then
+                unstable_count = unstable_count + 1
+                if srv_available then
+                    srv_status.status = _M.STATUS_ERR
+                    update_peer_status(srv_status, ups_sensi)
+                    srv_status.updated = true
+                end
+            end
+
+            if srv_status.updated ~= true then
+                tab_insert(ups_status, srv_status)
+            end
+        end
+    end
+
+    if next(ups_status) then
+        if error_count == server_count then
+            if ups_protected then
+                ups_status[1].status = _M.STATUS_UNSTABLE
+            end
+        elseif error_count + unstable_count == server_count then
+            tab_sort(ups_status, function(a, b) return a.status < b.status end)
+        end
+
+        update_upstream_status(ups_status, ups_sensi)
+    end
+end
+
+
+function _M.active_checkup(premature)
+    local ckey = base.CHECKUP_TIMER_KEY
+
+    update_time() -- flush cache time
+
+    if premature then
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+
+    for skey in pairs(base.upstream.checkups) do
+        cluster_heartbeat(skey)
+    end
+
+    local interval = base.upstream.checkup_timer_interval
+    local overtime = base.upstream.checkup_timer_overtime
+
+    state:set(base.CHECKUP_LAST_CHECK_TIME_KEY, localtime())
+    state:set(base.CHECKUP_TIMER_ALIVE_KEY, true, overtime)
+
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+
+    local ok, err = ngx.timer.at(interval, _M.active_checkup)
+    if not ok then
+        log(WARN, "failed to create timer: ", err)
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/round_robin.lua

@@ -0,0 +1,72 @@
+-- Copyright (C) 2014-2016, UPYUN Inc.
+
+local ceil = math.ceil
+
+local _M = { _VERSION = "0.11" }
+
+
+--[[
+parameters:
+    - (table) servers
+    - (function) peer_cb(index, server)
+return:
+    - (table) server
+    - (string) error
+--]]
+function _M.next_round_robin_server(servers, peer_cb)
+    local srvs_cnt = #servers
+
+    if srvs_cnt == 1 then
+        if peer_cb(1, servers[1]) then
+            return servers[1], nil
+        end
+
+        return nil, "round robin: no servers available"
+    end
+
+    -- select round robin server
+    local best
+    local max_weight
+    local weight_sum = 0
+    for idx = 1, srvs_cnt do
+        local srv = servers[idx]
+        -- init round robin state
+        srv.weight = srv.weight or 1
+        srv.effective_weight = srv.effective_weight or srv.weight
+        srv.current_weight = srv.current_weight or 0
+
+        if peer_cb(idx, srv) then
+            srv.current_weight = srv.current_weight + srv.effective_weight
+            weight_sum = weight_sum + srv.effective_weight
+
+            if srv.effective_weight < srv.weight then
+                srv.effective_weight = srv.effective_weight + 1
+            end
+
+            if not max_weight or srv.current_weight > max_weight then
+                max_weight = srv.current_weight
+                best = srv
+            end
+        end
+    end
+
+    if not best then
+        return nil, "round robin: no servers available"
+    end
+
+    best.current_weight = best.current_weight - weight_sum
+
+    return best, nil
+end
+
+
+function _M.free_round_robin_server(srv, failed)
+    if not failed then
+        return
+    end
+
+    srv.effective_weight = ceil((srv.effective_weight or 1) / 2)
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/try.lua

@@ -0,0 +1,248 @@
+-- Copyright (C) 2014-2016, UPYUN Inc.
+
+local cjson           = require "cjson.safe"
+local round_robin     = require "resty.checkups.round_robin"
+local consistent_hash = require "resty.checkups.consistent_hash"
+local base            = require "resty.checkups.base"
+
+local max        = math.max
+local sqrt       = math.sqrt
+local floor      = math.floor
+local tab_insert = table.insert
+local tostring   = tostring
+
+local update_time     = ngx.update_time
+local now             = ngx.now
+
+local _M = { _VERSION = "0.11" }
+
+local is_tab = base.is_tab
+
+local NEED_RETRY       = 0
+local REQUEST_SUCCESS  = 1
+local EXCESS_TRY_LIMIT = 2
+
+
+local function prepare_callbacks(skey, opts, upstream)
+    local ups = upstream or base.upstream.checkups[skey]
+
+    -- calculate count of cluster and server
+    local cls_keys = {}  -- string key or number level
+    local srvs_cnt = 0
+    if is_tab(opts.cluster_key) then  -- specify try cluster
+        for _, cls_key in ipairs(opts.cluster_key) do
+            local cls = ups.cluster[cls_key]
+            if is_tab(cls) then
+                tab_insert(cls_keys, cls_key)
+                srvs_cnt = srvs_cnt + #cls.servers
+            end
+        end
+    else  -- default try cluster
+        for cls_key, cls in pairs(ups.cluster) do
+            tab_insert(cls_keys, cls_key)
+            srvs_cnt = srvs_cnt + #cls.servers
+        end
+    end
+
+
+    -- get next level cluster
+    local cls_key
+    local cls_index = 0
+    local cls_cnt = #cls_keys
+    local next_cluster_cb = function()
+        cls_index = cls_index + 1
+        if cls_index > cls_cnt then
+            return
+        end
+
+        cls_key = cls_keys[cls_index]
+        return ups.cluster[cls_key]
+    end
+
+
+    -- get next select server
+    local mode = ups.mode
+    local next_server_func = round_robin.next_round_robin_server
+    local key
+    if mode ~= nil then
+        if mode == "hash" then
+            key = opts.hash_key or ngx.var.uri
+        elseif mode == "url_hash" then
+            key = ngx.var.uri
+        elseif mode == "ip_hash" then
+            key = ngx.var.remote_addr
+        end
+
+        next_server_func = consistent_hash.next_consistent_hash_server
+    end
+    local next_server_cb = function(servers, peer_cb)
+        return next_server_func(servers, peer_cb, key)
+    end
+
+
+    -- check whether ther server is available
+    local bad_servers = {}
+    local peer_cb = function(index, srv)
+        local key = ("%s:%s:%s:%s"):format(cls_key, srv.host, srv.port, srv.isp or "")
+        -- if bad_servers[key] then
+        --     return false
+        -- end
+
+        if ups.enable == false or (ups.enable == nil
+            and base.upstream.default_heartbeat_enable == false)
+        then
+            return base.get_srv_status(skey, srv, ups.id or "") == base.STATUS_OK
+        end
+
+        local peer_status = base.get_peer_status(skey, srv)
+        if (not peer_status or peer_status.status ~= base.STATUS_ERR)
+            and base.get_srv_status(skey, srv, ups.id or "") == base.STATUS_OK
+        then
+            return true
+        end
+    end
+
+    local retry_sleep = 0.05
+
+    -- check whether need retry
+    local statuses
+    if ups.typ == "http" and is_tab(ups.http_opts) then
+        statuses = ups.http_opts.statuses
+    end
+    local try_cnt = 0
+    local try_limit = opts.try or ups.try or srvs_cnt
+    local retry_cb = function(res)
+        if is_tab(res) and res.status and is_tab(statuses) then
+            if statuses[tostring(res.status)] ~= false then
+                return REQUEST_SUCCESS
+            end
+        elseif res then
+            return REQUEST_SUCCESS
+        end
+
+        try_cnt = try_cnt + 1
+        if try_cnt >= try_limit then
+            return EXCESS_TRY_LIMIT
+        end
+
+        ngx.sleep(try_cnt * retry_sleep)
+
+        return NEED_RETRY
+    end
+
+
+    -- check whether try_time has over amount_request_time
+    local try_time = 0
+    local try_time_limit = opts.try_timeout or ups.try_timeout or 0
+    local try_time_cb = function(this_time_try_time)
+        try_time = try_time + this_time_try_time
+        if try_time_limit == 0 then
+            return NEED_RETRY
+        elseif try_time >= try_time_limit then
+            return EXCESS_TRY_LIMIT
+        end
+
+        return NEED_RETRY
+    end
+
+
+    -- set some status
+    local free_server_func = round_robin.free_round_robin_server
+    if mode == "hash" then
+        free_server_func = consistent_hash.free_consitent_hash_server
+    end
+    local set_status_cb = function(srv, failed)
+        local key = ("%s:%s:%s:%s"):format(cls_key, srv.host, srv.port, srv.isp or "")
+        -- bad_servers[key] = failed
+        base.set_srv_status(skey, srv, ups.id or "", failed, ups)
+        free_server_func(srv, failed)
+    end
+
+
+    return {
+        next_cluster_cb = next_cluster_cb,
+        next_server_cb = next_server_cb,
+        retry_cb = retry_cb,
+        peer_cb = peer_cb,
+        set_status_cb = set_status_cb,
+        try_time_cb = try_time_cb,
+    }
+end
+
+
+local function copy_servers(servers)
+    local _servers = {}
+
+    for _, srv in ipairs(servers) do
+        table.insert(_servers, {host = srv.host, port = srv.port})
+    end
+
+    return _servers
+end
+
+
+--[[
+parameters:
+    - (string) skey
+    - (function) request_cb(host, port)
+    - (table) opts
+        - (number) try
+        - (table) cluster_key
+        - (string) hash_key
+return:
+    - (string) result
+    - (string) error
+--]]
+function _M.try_cluster(skey, request_cb, opts, upstream)
+    local callbacks = prepare_callbacks(skey, opts, upstream)
+
+    local next_cluster_cb = callbacks.next_cluster_cb
+    local next_server_cb  = callbacks.next_server_cb
+    local peer_cb         = callbacks.peer_cb
+    local retry_cb        = callbacks.retry_cb
+    local set_status_cb   = callbacks.set_status_cb
+    local try_time_cb     = callbacks.try_time_cb
+
+    -- iter servers function
+    local itersrvs = function(servers, peer_cb)
+        return function() return next_server_cb(servers, peer_cb) end
+    end
+
+    local res, err = nil, "no servers available"
+    repeat
+        -- get next level/key cluster
+        local cls = next_cluster_cb()
+        if not cls then
+            break
+        end
+
+        local loop_servers = copy_servers(cls.servers)
+        for srv, err in itersrvs(loop_servers, peer_cb) do
+            -- exec request callback by server
+            local start_time = now()
+            local args = opts.args or { srv }
+            res, err = request_cb(srv.host, srv.port, unpack(args))
+
+            local feedback = retry_cb(res)
+
+            -- check whether need retry
+            local end_time = now()
+            local delta_time = end_time - start_time
+
+            set_status_cb(srv, feedback ~= REQUEST_SUCCESS) -- set some status
+            if feedback ~= NEED_RETRY then
+                return res, err
+            end
+
+            local feedback_try_time = try_time_cb(delta_time)
+            if feedback_try_time ~= NEED_RETRY then
+                return nil, "try_timeout excceed"
+            end
+       end
+    until false
+
+    return res, err
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/uuid.lua

@@ -0,0 +1,340 @@
+--- jit-uuid
+-- Fast and dependency-free UUID library for LuaJIT/ngx_lua.
+-- @module jit-uuid
+-- @author Thibault Charbonnier
+-- @license MIT
+-- @release 0.0.5
+
+local bit = require 'bit'
+local tohex = bit.tohex
+local band = bit.band
+local bor = bit.bor
+
+local _M = {
+    _VERSION = '0.0.5'
+}
+
+----------
+-- seeding
+----------
+
+--- Seed the random number generator.
+-- Under the hood, this function calls `math.randomseed`.
+-- It makes sure to use the most appropriate seeding technique for
+-- the current environment, guaranteeing a unique seed.
+--
+-- To guarantee unique UUIDs, you must have correctly seeded
+-- the Lua pseudo-random generator (with `math.randomseed`).
+-- You are free to seed it any way you want, but this function
+-- can do it for you if you'd like, with some added guarantees.
+--
+-- @param[type=number] seed (Optional) A seed to use. If none given, will
+-- generate one trying to use the most appropriate technique.
+-- @treturn number `seed`: the seed given to `math.randomseed`.
+-- @usage
+-- local uuid = require 'resty.jit-uuid'
+-- uuid.seed()
+--
+-- -- in ngx_lua, seed in the init_worker context:
+-- init_worker_by_lua {
+--   local uuid = require 'resty.jit-uuid'
+--   uuid.seed()
+-- }
+function _M.seed(seed)
+    if not seed then
+        if ngx then
+            seed = ngx.time() + ngx.worker.pid()
+        elseif package.loaded['socket'] and package.loaded['socket'].gettime then
+            seed = package.loaded['socket'].gettime()*10000
+        else
+            seed = os.time()
+        end
+    end
+    math.randomseed(seed)
+    return seed
+end
+
+-------------
+-- validation
+-------------
+
+do
+    if ngx and string.find(ngx.config.nginx_configure(),'--with-pcre-jit',nil,true) then
+        local type = type
+        local re_find = ngx.re.find
+        local regex = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'
+
+        --- Validate a string as a UUID.
+        -- To be considered valid, a UUID must be given in its canonical
+        -- form (hexadecimal digits including the hyphen characters).
+        -- This function validates UUIDs disregarding their generation algorithm,
+        -- and in a case-insensitive manner, but checks the variant field.
+        --
+        -- Use JIT PCRE if available in OpenResty or fallbacks on Lua patterns.
+        --
+        -- @param[type=string] str String to verify.
+        -- @treturn boolean `valid`: true if valid UUID, false otherwise.
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- uuid.is_valid 'cbb297c0-a956-486d-ad1d-f9bZZZZZZZZZ' --> false
+        -- uuid.is_valid 'cbb297c0-a956-486d-dd1d-f9b42df9465a' --> false (invalid variant)
+        -- uuid.is_valid 'cbb297c0a956486dad1df9b42df9465a'     --> false (no dashes)
+        -- uuid.is_valid 'cbb297c0-a956-486d-ad1d-f9b42df9465a' --> true
+        function _M.is_valid(str)
+            -- it has proven itself efficient to first check the length with an
+            -- evenly distributed set of valid and invalid uuid lengths.
+            if type(str) ~= 'string' or #str ~= 36 then return false end
+            return re_find(str, regex, 'ioj') ~= nil
+        end
+
+    else
+        local match = string.match
+        local d = '[0-9a-fA-F]'
+        local p = '^'..table.concat({
+            d:rep(8),
+            d:rep(4),
+            d:rep(4),
+            '[89ab]'..d:rep(3),
+            d:rep(12)
+        }, '%-')..'$'
+
+        function _M.is_valid(str)
+            if type(str) ~= 'string' or #str ~= 36 then return false end
+            return match(str, p) ~= nil
+        end
+
+    end
+end
+
+----------------
+-- v4 generation
+----------------
+
+do
+    local fmt = string.format
+    local random = math.random
+
+    --- Generate a v4 UUID.
+    -- v4 UUIDs are created from randomly generated numbers.
+    --
+    -- @treturn string `uuid`: a v4 (randomly generated) UUID.
+    -- @usage
+    -- local uuid = require 'resty.jit-uuid'
+    --
+    -- local u1 = uuid()             ---> __call metamethod
+    -- local u2 = uuid.generate_v4()
+    function _M.generate_v4()
+        return fmt('%s%s%s%s-%s%s-%s%s-%s%s-%s%s%s%s%s%s',
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(bor(band(random(0, 255), 0x0F), 0x40), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(bor(band(random(0, 255), 0x3F), 0x80), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2))
+    end
+end
+
+----------------
+-- v3/v5 generation
+----------------
+
+do
+    if ngx then
+        local tonumber = tonumber
+        local gmatch = string.gmatch
+        local type = type
+        local char = string.char
+        local fmt = string.format
+        local sub = string.sub
+
+        local function factory(namespace, hash_fn)
+            if not _M.is_valid(namespace) then
+                return nil, 'namespace must be a valid UUID'
+            end
+
+            local binary = ''
+            local iter = gmatch(namespace, '([%a%d][%a%d])') -- pattern faster than PCRE without resty.core
+            while true do
+                local m = iter()
+                if not m then break end
+                binary = binary..char(tonumber(m, 16)) -- no noticeable improvement with buffer table
+            end
+
+            return function(name)
+                if type(name) ~= 'string' then
+                    return nil, 'name must be a string'
+                end
+
+                local hash, ver, var = hash_fn(binary, name)
+
+                return fmt('%s-%s-%s%s-%s%s-%s', sub(hash, 1, 8),
+                sub(hash, 9, 12),
+                ver,
+                sub(hash, 15, 16),
+                var,
+                sub(hash, 19, 20),
+                sub(hash, 21, 32))
+            end
+        end
+
+        --- Instanciate a v3 UUID factory.
+        -- @function factory_v3
+        -- Creates a closure generating namespaced v3 UUIDs.
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @treturn function `factory`: a v3 UUID generator.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local fact = assert(uuid.factory_v3('e6ebd542-06ae-11e6-8e82-bba81706b27d'))
+        --
+        -- local u1 = fact('hello')
+        -- ---> 3db7a435-8c56-359d-a563-1b69e6802c78
+        --
+        -- local u2 = fact('foobar')
+        -- ---> e8d3eeba-7723-3b72-bbc5-8f598afa6773
+        do
+            local md5 = ngx.md5
+
+            local function v3_hash(binary, name)
+                local hash = md5(binary..name)
+                return  hash,
+                tohex(bor(band(tonumber(sub(hash, 13, 14), 16), 0x0F), 0x30), 2),
+                tohex(bor(band(tonumber(sub(hash, 17, 18), 16), 0x3F), 0x80), 2)
+            end
+
+            function _M.factory_v3(namespace)
+                return factory(namespace, v3_hash)
+            end
+        end
+
+        --- Instanciate a v5 UUID factory.
+        -- @function factory_v5
+        -- Creates a closure generating namespaced v5 UUIDs.
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @treturn function `factory`: a v5 UUID generator.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local fact = assert(uuid.factory_v5('e6ebd542-06ae-11e6-8e82-bba81706b27d'))
+        --
+        -- local u1 = fact('hello')
+        -- ---> 4850816f-1658-5890-8bfd-1ed14251f1f0
+        --
+        -- local u2 = fact('foobar')
+        -- ---> c9be99fc-326b-5066-bdba-dcd31a6d01ab
+        do
+            local ffi = require 'ffi'
+
+            local sha1_bin = ngx.sha1_bin
+            local C = ffi.C
+            local ffi_new = ffi.new
+            local ffi_str = ffi.string
+            local str_type = ffi.typeof('uint8_t[?]')
+            ffi.cdef [[
+            typedef unsigned char u_char;
+            u_char * ngx_hex_dump(u_char *dst, const u_char *src, size_t len);
+            ]]
+
+            local function bin_tohex(s)
+                local len = #s * 2
+                local buf = ffi_new(str_type, len)
+                C.ngx_hex_dump(buf, s, #s)
+                return ffi_str(buf, len)
+            end
+
+            local function v5_hash(binary, name)
+                local hash = bin_tohex(sha1_bin(binary..name))
+                return  hash,
+                tohex(bor(band(tonumber(sub(hash, 13, 14), 16), 0x0F), 0x50), 2),
+                tohex(bor(band(tonumber(sub(hash, 17, 18), 16), 0x3F), 0x80), 2)
+            end
+
+            function _M.factory_v5(namespace)
+                return factory(namespace, v5_hash)
+            end
+        end
+
+        --- Generate a v3 UUID.
+        -- v3 UUIDs are created from a namespace and a name (a UUID and a string).
+        -- The same name and namespace result in the same UUID. The same name and
+        -- different namespaces result in different UUIDs, and vice-versa.
+        -- The resulting UUID is derived using MD5 hashing.
+        --
+        -- This is a sugar function which instanciates a short-lived v3 UUID factory.
+        -- It is an expensive operation, and intensive generation using the same
+        -- namespaces should prefer allocating their own long-lived factory with
+        -- `factory_v3`.
+        --
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @param[type=string] name
+        -- @treturn string `uuid`: a v3 (namespaced) UUID.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local u = uuid.generate_v3('e6ebd542-06ae-11e6-8e82-bba81706b27d', 'hello')
+        -- ---> 3db7a435-8c56-359d-a563-1b69e6802c78
+        function _M.generate_v3(namespace, name)
+            local fact, err = _M.factory_v3(namespace)
+            if not fact then return nil, err end
+
+            return fact(name)
+        end
+
+        --- Generate a v5 UUID.
+        -- v5 UUIDs are created from a namespace and a name (a UUID and a string).
+        -- The same name and namespace result in the same UUID. The same name and
+        -- different namespaces result in different UUIDs, and vice-versa.
+        -- The resulting UUID is derived using SHA-1 hashing.
+        --
+        -- This is a sugar function which instanciates a short-lived v5 UUID factory.
+        -- It is an expensive operation, and intensive generation using the same
+        -- namespaces should prefer allocating their own long-lived factory with
+        -- `factory_v5`.
+        --
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @param[type=string] name
+        -- @treturn string `uuid`: a v5 (namespaced) UUID.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local u = uuid.generate_v5('e6ebd542-06ae-11e6-8e82-bba81706b27d', 'hello')
+        -- ---> 4850816f-1658-5890-8bfd-1ed14251f1f0
+        function _M.generate_v5(namespace, name)
+            local fact, err = _M.factory_v5(namespace)
+            if not fact then return nil, err end
+
+            return fact(name)
+        end
+
+    else
+
+        function _M.factory_v3() error('v3 UUID generation only supported in ngx_lua', 2) end
+        function _M.generate_v3() error('v3 UUID generation only supported in ngx_lua', 2) end
+        function _M.factory_v5() error('v5 UUID generation only supported in ngx_lua', 2) end
+        function _M.generate_v5() error('v5 UUID generation only supported in ngx_lua', 2) end
+
+    end
+end
+
+return setmetatable(_M, {
+    __call = _M.generate_v4
+})

curve-snapshotcloneserver-nginx/app/src/init.lua

@@ -0,0 +1,21 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+require "resty.core"
+local checkups = require "resty.checkups.api"
+config = require "config"
+
+config.global.version = "v0.0.1"
+checkups.init(config)

curve-snapshotcloneserver-nginx/app/src/init_worker.lua

@@ -0,0 +1,24 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+local uuid     = require 'resty.uuid'
+local checkups = require "resty.checkups.api"
+
+local config = config
+
+checkups.prepare_checker(config)
+checkups.create_checker()
+
+uuid.seed()

curve-snapshotcloneserver-nginx/app/src/modules/httpipe.lua

@@ -0,0 +1,964 @@
+-- Copyright (C) Monkey Zhang (timebug), UPYUN Inc.
+
+
+local type = type
+local error = error
+local pairs = pairs
+local ipairs = ipairs
+local rawset = rawset
+local rawget = rawget
+local sub = string.sub
+local gsub = string.gsub
+local find = string.find
+local tostring = tostring
+local tonumber = tonumber
+local tcp = ngx.socket.tcp
+local match = string.match
+local upper = string.upper
+local lower = string.lower
+local concat = table.concat
+local insert = table.insert
+local format = string.format
+local setmetatable = setmetatable
+local ngx_re_match = ngx.re.match
+local encode_args = ngx.encode_args
+local ngx_req_socket = ngx.req.socket
+local ngx_req_get_headers = ngx.req.get_headers
+
+
+local _M = { _VERSION = "0.08" }
+
+--------------------------------------
+-- LOCAL CONSTANTS                  --
+--------------------------------------
+
+local mt = { __index = _M }
+
+local HTTP = {
+    [11] = " HTTP/1.1\r\n",
+    [10] = " HTTP/1.0\r\n"
+}
+
+local PORT = {
+    http = 80,
+    https = 443
+}
+
+local USER_AGENT = "Resty/HTTPipe-" .. _M._VERSION
+
+local STATE_NOT_READY = 0
+local STATE_BEGIN = 1
+local STATE_READING_HEADER = 2
+local STATE_READING_BODY = 3
+local STATE_EOF = 4
+
+local common_headers = {
+    "Cache-Control",
+    "Content-Length",
+    "Content-Type",
+    "Date",
+    "ETag",
+    "Expires",
+    "Host",
+    "Location",
+    "User-Agent"
+}
+
+for _, key in ipairs(common_headers) do
+    rawset(common_headers, key, key)
+    rawset(common_headers, lower(key), key)
+end
+
+local state_handlers
+
+--------------------------------------
+-- HTTP BASE FUNCTIONS              --
+--------------------------------------
+
+local function normalize_header(key)
+    local val = common_headers[key]
+    if val then
+        return val
+    end
+    key = lower(key)
+    val = common_headers[lower(key)]
+    if val then
+        return val
+    end
+
+    key = gsub(key, "^%l", upper)
+    key = gsub(key, "-%l", upper)
+    return key
+end
+
+
+local function req_header(self, opts)
+    self.method = upper(opts.method or "GET")
+
+    local req = {
+        self.method,
+        " "
+    }
+
+    local path = opts.path
+    if type(path) ~= "string" then
+        path = "/"
+    elseif sub(path, 1, 1) ~= "/" then
+        path = "/" .. path
+    end
+    insert(req, path)
+
+    if type(opts.query) == "table" then
+        opts.query = encode_args(opts.query)
+    end
+
+    if type(opts.query) == "string" then
+        insert(req, "?" .. opts.query)
+    end
+
+    insert(req, HTTP[opts.version])
+
+    opts.headers = opts.headers or {}
+
+    local headers = {}
+
+    for k, v in pairs(opts.headers) do
+        if opts.unnormalize_header then
+            headers[k] = v
+        else
+            headers[normalize_header(k)] = v
+        end
+    end
+
+    if type(opts.body) == "string" then
+        headers["Content-Length"] = #opts.body
+    elseif self.previous.content_length and
+    self.previous.content_length >= 0 then
+        headers["Content-Length"] = self.previous.content_length
+    end
+
+    if type(opts.body) == "function" and
+    not headers["Content-Length"] and not headers["Transfer-Encoding"] then
+        headers["Transfer-Encoding"] = "chunked"
+    end
+
+    if not headers["Host"] then
+        headers["Host"] = self.host
+    end
+
+    if not headers["User-Agent"] then
+        headers["User-Agent"] = USER_AGENT
+    end
+
+    if not headers["Accept"] then
+        headers["Accept"] = "*/*"
+    end
+
+    if opts.version == 10 and not headers["Connection"] then
+        headers["Connection"] = "keep-alive"
+    end
+
+    for key, values in pairs(headers) do
+        if type(values) ~= "table" then
+            values = { values }
+        end
+
+        key = tostring(key)
+        for _, value in pairs(values) do
+            insert(req, key .. ": " .. tostring(value) .. "\r\n")
+        end
+    end
+
+    insert(req, "\r\n")
+
+    return concat(req), headers
+end
+
+
+-- local scheme, host, port, path, args = unpack(_M:parse_uri(uri))
+function _M.parse_uri(self, uri)
+    local r = [[^(https?)://([^:/]+)(?::(\d+))?(.*)]]
+    local m, err = ngx_re_match(uri, r, "jo")
+    if not m then
+        return nil, err or "bad uri"
+    end
+
+    if not m[3] then m[3] = PORT[m[1]] end
+    if not m[4] then
+        m[4] = "/"
+    else
+        local raw = m[4]
+        local from = find(raw, "?")
+        if from then
+            m[4] = raw:sub(1, from - 1) -- path
+            m[5] = raw:sub(from + 1)    -- args
+        end
+    end
+
+    return m
+end
+
+
+local function init(self)
+    self.total_size = 0
+    self.state = STATE_NOT_READY
+    self.chunked = false
+    self.keepalive = true
+    self._eof = false
+    self.previous = {}
+    self.remaining = nil
+
+    return self
+end
+
+
+--------------------------------------
+-- HTTP PIPE FUNCTIONS              --
+--------------------------------------
+
+-- local hp, err = _M:new(chunk_size?, sock?)
+function _M.new(self, chunk_size, sock)
+    if not sock then
+        local s, err = tcp()
+        if not s then
+            return nil, err
+        end
+        sock = s
+    end
+
+    return setmetatable(init({
+        sock = sock,
+        chunk_size = chunk_size or 8192,
+    }), mt)
+end
+
+
+-- local ok, err = _M:set_timeout(time)
+function _M.set_timeout(self, time)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    return sock:settimeout(time)
+end
+
+
+-- local session, err = _M:ssl_handshake(self, ...)
+function _M.ssl_handshake(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    if not ngx.config
+        or not ngx.config.ngx_lua_version
+        or ngx.config.ngx_lua_version < 9011
+    then
+        error("ngx_lua 0.9.11+ required")
+    end
+
+    return sock:sslhandshake(...)
+end
+
+
+-- local ok, err = _M:connect(self, host, port, opts?)
+-- local ok, err = _M:connect("unix:/path/to/unix-domain.socket", opts?)
+function _M.connect(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    self.host = select(1, ...)
+    if sub(self.host, 1, 5) == "unix:" then
+        -- https://tools.ietf.org/html/rfc2616#section-14.23
+        -- A client MUST include a Host header field in all HTTP/1.1 request
+        -- messages . If the requested URI does not include an Internet host
+        -- name for the service being requested, then the Host header field MUST
+        -- be given with an empty value.
+        self.host = ""
+    end
+
+    self.port = select(2, ...)
+    if type(self.port) == "string" then
+        self.port = tonumber(self.port)
+    elseif type(self.port) ~= "number" then
+        self.port = nil
+    end
+
+    return sock:connect(...)
+end
+
+
+local function discard_line(self)
+    local read_line = self.read_line
+
+    local line, err = read_line()
+    if not line then
+        return nil, err
+    end
+
+    return 1
+end
+
+
+local function should_receive_body(method, code)
+    if method == "HEAD" then return nil end
+    if code == 204 or code == 304 then return nil end
+    if code >= 100 and code < 200 then return nil end
+    return true
+end
+
+
+local function read_body_part(self)
+    if not self.is_req_socket and
+    not should_receive_body(self.method, self.status_code) then
+        self.state = STATE_EOF
+        return 'body_end', nil
+    end
+
+    local sock = self.sock
+    local remaining = self.remaining
+    local chunk_size = self.chunk_size
+
+    if self.maxsize and remaining and remaining > self.maxsize then
+        return nil, nil, "exceeds maxsize"
+    end
+
+    if self.chunked == true and
+    (remaining == nil or remaining == 0) then
+        local read_line = self.read_line
+        local data, err = read_line()
+
+        if err then
+            return nil, nil, err
+        end
+
+        if data == "" then
+            data, err = read_line()
+            if err then
+                return nil, nil, err
+            end
+        end
+
+        if data then
+            if data == "0" then
+                local ok, err = discard_line(self)
+                if not ok then
+                    return nil, nil, err
+                end
+
+                self.state = STATE_EOF
+                return 'body_end', nil
+            else
+                local length = tonumber(data, 16)
+                remaining = length
+            end
+        end
+    end
+
+    if remaining == 0 then
+        self.state = STATE_EOF
+        return 'body_end', nil
+    end
+
+    if remaining ~= nil and remaining < chunk_size then
+        chunk_size = remaining
+    end
+
+    local chunk, err, partial = sock:receive(chunk_size)
+
+    local data = ""
+    if not err then
+        if chunk then
+            data = chunk
+        end
+    elseif err == "closed" then
+        self.state = STATE_EOF
+        if partial then
+            chunk_size = #partial
+            if remaining and remaining - chunk_size ~= 0 then
+                return nil, partial, err
+            end
+
+            data = partial
+        else
+            return 'body_end', nil
+        end
+    else
+        return nil, nil, err
+    end
+
+    if remaining ~= nil then
+        self.remaining = remaining - chunk_size
+        self.total_size = self.total_size + chunk_size
+
+        if self.maxsize and self.total_size > self.maxsize then
+            return nil, nil, "exceeds maxsize"
+        end
+    end
+
+    return 'body', data
+end
+
+
+local function read_header_part(self)
+    local read_line = self.read_line
+
+    local line, err = read_line()
+    if not line then
+        return nil, nil, err
+    end
+
+    if line == "" then
+        if self.chunked then
+            self.remaining = nil
+        end
+
+        self.state = STATE_READING_BODY
+        return 'header_end', nil
+    end
+
+    local m, err = ngx_re_match(line, [[^(.+?):\s*(.+)]], "jo")
+    if not m then
+        return 'header', line
+    end
+
+    local name, value = m[1], m[2]
+
+    local vname = lower(name)
+    if vname == "content-length" then
+        self.remaining = tonumber(value)
+    end
+
+    if vname == "transfer-encoding" and value ~= "identity" then
+        self.chunked = true
+    end
+
+    if vname == "connection" and value == "close" then
+        self.keepalive = value ~= "close"
+    end
+
+    return 'header', { normalize_header(name), value, line }
+end
+
+
+local function read_statusline(self)
+    local sock = self.sock
+    if self.read_line == nil then
+        local rl, err = sock:receiveuntil("\n")
+        if not rl then
+            return nil, nil, err
+        end
+        self.read_line = function()
+            --[[
+            The line terminator for message-header fields is the sequence CRLF.
+            However, we recommend that applications, when parsing such headers,
+            recognize a single LF as a line terminator and ignore the leading
+            CR.
+            REF: http://stackoverflow.com/questions/5757290/http-header-line-break-style
+            --]]
+            local data, err = rl()
+            if data and data:sub(-1) == "\r" then
+                data = data:sub(1, -2)
+            end
+
+            return data, err
+        end
+    end
+
+    local read_line = self.read_line
+
+    local line, err = read_line()
+    if not line then
+        return nil, nil, err
+    end
+
+    local version, status = match(line, "HTTP/(%d*%.%d*) (%d%d%d)")
+    if not version or not status then
+        -- return nil, nil, "not match statusline"
+        return nil, nil, line
+    end
+
+    version = tonumber(version) * 10
+    if version < 11 then
+        self.keepalive = false
+    end
+
+    self.status_code = tonumber(status)
+
+    if self.status_code == 100 then
+        local ok, err = discard_line(self)
+        if not ok then
+            return nil, nil, err
+        end
+
+        self.state = STATE_BEGIN
+    else
+        self.state = STATE_READING_HEADER
+    end
+
+    return 'statusline', status
+end
+
+
+-- local ok, err = _M:set_keepalive(...)
+function _M.set_keepalive(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    self._eof = true
+
+    if self.keepalive then
+        return sock:setkeepalive(...)
+    end
+
+    return sock:close()
+end
+
+
+-- local times, err = _M:get_reused_times()
+function _M.get_reused_times(self)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    return sock:getreusedtimes()
+end
+
+
+-- local ok, err = _M:close()
+function _M.close(self)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    self._eof = true
+
+    return sock:close()
+end
+
+
+local function read_eof(self)
+    if not self.is_req_socket then
+        self:set_keepalive()
+    end
+    return 'eof', nil
+end
+
+
+-- local typ, res, err = _M:read(chunk_size?)
+function _M.read(self, chunk_size)
+    local sock = self.sock
+    if not sock then
+        return nil, nil, "not initialized"
+    end
+
+    if chunk_size then
+        self.chunk_size = chunk_size
+    end
+
+    if self.state == STATE_NOT_READY then
+        return nil, nil, "not ready"
+    end
+
+    if self.read_timeout then
+        sock:settimeout(self.read_timeout)
+    end
+
+    local handler = state_handlers[self.state]
+    if handler then
+        return handler(self)
+    end
+
+    return nil, nil, "bad state: " .. self.state
+end
+
+
+-- local eof = _M:eof()
+function _M.eof(self)
+    return self._eof
+end
+
+
+state_handlers = {
+    read_statusline,
+    read_header_part,
+    read_body_part,
+    read_eof
+}
+
+
+-- local res, err = _M:read_response(callback?)
+function _M.read_response(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    local callback = ...
+    if type(callback) ~= "table" then
+        callback = {}
+    end
+
+    local status
+    local headers = {}
+    local chunks = {}
+
+    while not self._eof do
+        local typ, res, err = self:read()
+        if not typ then
+            return nil, err
+        end
+
+        if typ == 'statusline' then
+            status = tonumber(res)
+        end
+
+        if typ == 'header' then
+            if type(res) == "table" then
+                local key = res[1]
+                if headers[key] then
+                    if type(headers[key]) ~= "table" then
+                        headers[key] = { headers[key] }
+                    end
+                    insert(headers[key], tostring(res[2]))
+                else
+                    headers[key] = tostring(res[2])
+                end
+            end
+        end
+
+        if typ == 'header_end' then
+            if callback.header_filter then
+                local rc = callback.header_filter(status, headers)
+                if rc then break end
+            end
+        end
+
+        if typ == 'body' then
+            if callback.body_filter then
+                local rc = callback.body_filter(res)
+                if rc then break end
+            else
+                insert(chunks, res)
+            end
+        end
+
+        if typ == 'eof' then
+            break
+        end
+    end
+
+    return { status = status, headers = headers, body = concat(chunks) }
+end
+
+
+-- local ok, err = _M:send_request(opts?)
+function _M.send_request(self, opts)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    if opts.send_timeout then
+        sock:settimeout(opts.send_timeout)
+    end
+
+    if opts.read_timeout then
+        self.read_timeout = opts.read_timeout
+    end
+
+    if opts.version and not HTTP[opts.version] then
+        return nil, "unknown HTTP version"
+    else
+        opts.version = opts.version or 11
+    end
+
+    local req, headers = req_header(self, opts)
+    local bytes, err = sock:send(req)
+    if not bytes then
+        return nil, err
+    end
+
+    if type(opts.body) == "string" then
+        local bytes, err = sock:send(opts.body)
+        if not bytes then
+            return nil, err
+        end
+    elseif type(opts.body) == "function" then
+        local chunked = headers["Transfer-Encoding"] == "chunked"
+        repeat
+            local chunk, err = opts.body()
+            if chunk then
+                local size = #chunk
+                if chunked and size > 0 then
+                    chunk = concat({
+                            format("%X", size), "\r\n",
+                            chunk, "\r\n",
+                    })
+                end
+                local bytes, err = sock:send(chunk)
+                if not bytes then
+                    return nil, err
+                end
+            elseif err then
+                return nil, err
+            end
+        until not chunk
+        if chunked then
+            local bytes, err = sock:send("0\r\n\r\n")
+            if not bytes then
+                return nil ,err
+            end
+        end
+    end
+
+    self.maxsize = opts.maxsize
+    self.state = STATE_BEGIN
+
+    return 1
+end
+
+
+local function get_body_reader(self)
+    return function (chunk_size)
+        local typ, res, err = self:read(chunk_size)
+        if not typ then
+            return nil, err
+        end
+
+        if typ == 'body' then
+            return res
+        end
+
+        if typ == 'body_end' then
+            read_eof(self)
+        end
+
+        return
+    end
+end
+
+
+local function redirect(self, url, opts)
+    if sub(url, 1, 1) == "/" then
+        url = ("http://%s:%s%s"):format(gsub(self.host, [[:.+$]], ""), self.port, url)
+    end
+
+    local parsed_uri, _ = self:parse_uri(url)
+    if not parsed_uri then
+        return nil, "invalid redirect url"
+    end
+
+    local scheme, host, port, path, args = unpack(parsed_uri)
+    opts.path = path
+    opts.query = args
+
+    if scheme == "https" and host ~= opts.req_host then
+        opts.ssl_enable = true
+    else
+        opts.ssl_enable = false
+    end
+
+    if type(opts.is_valid_addr) == "function" then
+        local valid, format = opts.is_valid_addr(host)
+        if not valid then
+            return nil, "invalid redirect host"
+        end
+
+        -- 0 IP_FORMAT
+        -- 1 DOMAIN_FORMAT
+        if format == 1 then
+            opts.headers["Host"] = host
+        end
+    end
+
+    if opts.stream then
+        local res, err = self:read_response()
+        if not res then
+            return nil, err
+        end
+    end
+
+    init(self)
+
+    opts.redirect_follow_num = opts.redirect_follow_num - 1
+    if opts.redirect_follow_num <= 0 then
+        opts.allow_redirects = false
+    end
+
+    if host == opts.req_host then
+        host = gsub(self.host, [[:.+$]], "")
+        port = self.port
+    end
+
+    return self:request(host, port, opts)
+end
+
+
+-- local res, err = _M:request(opts?)
+-- local res, err = _M:request(host, port, opts?)
+-- local res, err = _M:request("unix:/path/to/unix-domain.socket", opts?)
+function _M.request(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    local arguments = {...}
+    local n = #arguments
+    if n > 3 then
+        return nil, "expecting 0, 1, 2, or 3 arguments, but seen " ..
+            tostring(n)
+    end
+
+    local opts = {}
+    if type(arguments[n]) == "table" then
+        opts = arguments[n]
+        arguments[n] = nil
+    end
+
+    if not opts.headers then
+        opts.headers = {}
+    end
+
+    if n > 0 and arguments[1] then
+        local rc, err = self:connect(unpack(arguments))
+        if not rc then
+            return nil, err
+        end
+
+        if self.port then
+            if opts.ssl_enable and self.port ~= 443 then
+                self.host = self.host .. ":" .. tostring(self.port)
+            elseif not opts.ssl_enable and self.port ~= 80 then
+                self.host = self.host .. ":" .. tostring(self.port)
+            end
+        end
+
+        if opts.ssl_enable then
+            local server_name = opts.headers["Host"] or self.host
+            if opts.ssl_server_name then
+                server_name = opts.ssl_server_name
+            end
+
+            local ssl_verify = true
+            if opts.ssl_verify == false then
+                ssl_verify = false
+            end
+
+            local ok, err = self:ssl_handshake(nil, server_name, ssl_verify)
+            if not ok then
+                return nil, err
+            end
+         end
+    end
+
+    local prev = self.previous.pipe
+    if self.previous.body_reader then
+        opts.body = self.previous.body_reader
+    end
+
+    local ok, err = self:send_request(opts)
+    if not ok then
+        if prev then prev:close() end
+        return nil, err
+    end
+
+    local res, err = self:read_response{
+        header_filter = function (status, headers)
+            return opts.stream
+        end
+    }
+
+    if res and opts.stream then
+        res.body_reader = get_body_reader(self)
+
+        local size = tonumber(res.headers["Content-Length"]) or -1 -- -1:chunked
+        local pipe, err = self:new(self.chunk_size)
+        if not pipe then
+            return nil, err
+        end
+
+        pipe.previous.pipe = self
+        pipe.previous.content_length = size
+        pipe.previous.body_reader = res.body_reader
+
+        res.pipe = pipe
+    end
+
+    if res and opts.allow_redirects
+        and (self.method == "GET" or self.method == "HEAD")
+        and (res.status == 301 or res.status == 302)
+    then
+        local url = res.headers["Location"]
+        if type(url) == "string" then
+            local _res, _err = redirect(self, url, opts)
+            if _res then
+                return _res, _err
+            end
+        end
+    end
+
+    return res, err
+end
+
+
+-- local res, err = _M:request_uri(uri, opts?)
+function _M.request_uri(self, uri, opts)
+    if not opts then
+        opts = {}
+    end
+
+    local parsed_uri, err = self:parse_uri(uri)
+    if not parsed_uri then
+        return nil, err
+    end
+
+    local scheme, host, port, path, args = unpack(parsed_uri)
+    if not opts.path then
+        opts.path = path
+    end
+
+    if not opts.query then
+        opts.query = args
+    end
+
+    if scheme == "https" then
+        opts.ssl_enable = true
+    end
+
+    return self:request(host, port, opts)
+end
+
+
+-- local reader, err = _M:get_client_body_reader(chunk_size?)
+function _M.get_client_body_reader(self, chunk_size)
+    local sock, err = ngx_req_socket()
+
+    if not sock then
+        return nil, err
+    end
+
+    local hp = self:new(chunk_size, sock)
+    local headers = ngx_req_get_headers()
+
+    hp.chunked = headers["Transfer-Encoding"] == "chunked"
+    if not hp.chunked then
+        hp.remaining = tonumber(headers["Content-Length"])
+    end
+
+    hp.state = STATE_READING_BODY
+    hp.is_req_socket = 1
+
+    return get_body_reader(hp)
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/modules/httproxy.lua

@@ -0,0 +1,392 @@
+-- Copyright (C) 2014 Jing Ye (yejingx), UPYUN Inc.
+
+local checkups = require "resty.checkups"
+local httpipe = require "modules.httpipe"
+local reqlimit = require "modules.reqlimit"
+local utils = require "modules.utils"
+
+local type = type
+local tostring = tostring
+local str_format = string.format
+local insert = table.insert
+local concat = table.concat
+local cjson = require "cjson.safe"
+
+
+local req_start_time = ngx.req.start_time
+local ngx_now = ngx.now
+local send_headers = ngx.send_headers
+
+
+local _M = { _VERSION = "0.10" }
+
+local function is_func(f)
+    return type(f) == "function"
+end
+
+local function set_uptime()
+    if ngx.var.uptime then
+        ngx.var.uptime = str_format("%.3f", ngx_now() - req_start_time())
+    end
+end
+
+
+local function set_xstate(errmsg)
+    if ngx.var.xstate then
+        ngx.var.xstate = tostring(errmsg)
+    end
+end
+
+
+local function set_upinfo(node, code, time, ctx)
+    local append_var = function(k, v)
+        if not v then
+            return
+        end
+
+        local value  = ngx.var[k]
+        if not value then
+            return
+        end
+
+        ngx.var[k] = value == '-' and v or value .. ", " .. v
+    end
+
+    if code and time and ctx.upstats == true and ctx.host and ctx.port then
+        local upstat = str_format("%s:%s/%s/%s", ctx.host, ctx.port, code, time)
+        append_var("upstats", upstat)
+    end
+
+    local srv = ctx.srv
+    if node and srv and srv.isp then
+        node = ("%s/%s"):format(srv.isp, node)
+    end
+
+    if node and srv and srv.domain then
+        node = ("%s-%s"):format(srv.domain, node)
+    end
+
+    append_var("upnode", node)
+    append_var("uptimes", time)
+    append_var("upcode", code)
+
+    if node and code and time and ngx.var.http_x_debug then
+        local upstat = str_format("%s/%s/%s", node, code, time)
+        append_var("uptrace", upstat)
+    end
+end
+
+
+function _M.close(hp)
+    if hp and not hp:eof() then
+        local ok, err = hp:close()
+        if not ok then -- warn: unread data in buffer
+            ngx.log(ngx.WARN, "failed to close: ", err)
+        end
+    end
+end
+
+
+local function prepare_opts(method, uri, source, opts)
+    local client_hp, err = httpipe:new()
+    if not client_hp then
+        return nil, err
+    end
+
+    local tm, send_tm, read_tm = checkups.get_ups_timeout(source)
+    local timeout = opts.timeout or tm or 5
+    local send_timeout = opts.send_timeout or send_tm or timeout
+    local read_timeout = opts.read_timeout or read_tm or timeout
+
+    local calc_timeout = opts.calc_timeout
+    if is_func(calc_timeout) then
+        timeout, send_timeout, read_timeout = calc_timeout(reqlimit.STATE_REQUEST, opts.slice)
+        if not timeout or not send_timeout or not read_timeout then
+            return nil, reqlimit.TIME_RUN_OUT
+        end
+    end
+
+    local method_with_request_body = {
+        --__FORWARD_GET_PAYLOAD__
+        GET = true, PUT = true, POST = true, DELETE = true, PATCH = true,
+        --__FORWARD_GET_PAYLOAD__
+    }
+    if not opts.body and method_with_request_body[method] then
+        opts.body = client_hp:get_client_body_reader(opts.chunk_size or 8192)
+    end
+
+    local req_opts = {
+        method = method, path = uri,
+        allow_redirects = opts.allow_redirects,
+        redirect_follow_num = opts.redirect_follow_num,
+        is_valid_addr = opts.is_valid_addr,
+        ssl_enable = opts.ssl_enable,
+        ssl_server_name = opts.ssl_server_name,
+        ssl_verify = opts.ssl_verify or false,
+        headers = opts.headers or {}, query = opts.query,
+        unnormalize_header = opts.unnormalize_header,
+        timeout = timeout * 1000,
+        send_timeout = send_timeout * 1000,
+        read_timeout = read_timeout * 1000,
+        body = opts.body,
+        maxsize = opts.max_body_size,
+        stream = true,
+        req_host = opts.req_host
+    }
+
+    return req_opts, nil
+end
+
+
+local function do_request(host, port, ctx, req_opts)
+    local res, err, hp
+    hp, err = httpipe:new()
+    if not hp then
+        return nil, err
+    end
+
+    ctx.hp = hp
+
+    local start = ngx.now()
+    hp:set_timeout(req_opts.timeout)
+
+    res, err = hp:request(host, port, req_opts)
+
+    ngx.log(ngx.INFO, "request ", host, ":", tostring(port), " ", cjson.encode(req_opts))
+
+    if not res then
+        ngx.log(ngx.WARN, "failed to request: ", host, ":", tostring(port), " ", err)
+    end
+
+    local uptime = ("%.3f"):format(ngx_now() - start)
+    local upcode = res and res.status or (err == "timeout" and err or "err")
+    local upnode = ("%s:%s"):format(host, port)
+
+    set_upinfo(upnode, upcode, uptime, ctx)
+
+    return res, err
+end
+
+
+local function capture(method, uri, source, opts)
+    opts = opts or {}
+    local req_opts, err = prepare_opts(method, uri, source, opts)
+    if not req_opts then
+        return nil, nil, err
+    end
+
+    local res
+    local calc_timeout = opts.calc_timeout
+
+    -- set_upinfo
+    local ctx = {
+        host = opts.host, port = opts.port,
+        srv = opts.srv, upstats = opts.upstats
+    }
+
+    if source then
+        res, err = checkups.ready_ok(source, do_request, {
+            args = { ctx, req_opts }, cluster_key = opts.cluster_key
+        })
+    else
+        res, err = do_request(opts.host, opts.port, ctx, req_opts)
+    end
+
+    local hp = ctx.hp
+    local failed_filter = opts.failed_capture_filter
+    local post_filter = opts.post_capture_filter
+
+    if not res then
+        if failed_filter then
+            failed_filter(source, err)
+        end
+        _M.close(hp)
+        return nil, nil, err
+    end
+
+    if post_filter then
+        post_filter(source, res)
+    end
+
+    res.headers["Transfer-Encoding"] = nil
+    res.headers["Connection"] = nil
+
+    local discard_abnormal_body = opts.discard_abnormal_body
+
+    local should_receive_body = true
+    if discard_abnormal_body and (method ~= "GET"
+        or (res.status ~= 200 and res.status ~= 206))
+    then
+        should_receive_body = false
+    end
+
+    if is_func(calc_timeout) then
+        local read_timeout = calc_timeout(reqlimit.STATE_READ)
+        if not read_timeout then
+            _M.close(hp)
+            return nil, nil, reqlimit.TIME_RUN_OUT
+        end
+
+        hp.read_timeout = read_timeout * 1000
+    end
+
+    if not should_receive_body or type(res.status) ~= "number" then
+        hp:read_response{ body_filter = function (chunk) end }
+        _M.close(hp)
+    end
+
+
+    return res, hp, nil
+end
+
+
+function _M.capture(method, uri, source, opts)
+    local res, hp, err = capture(method, uri, source, opts)
+
+    -- when POST failed(upstream 502/504/503/error)
+    -- discard req body explicitly
+    if (not res or (res and tonumber(res.status or 0) >= 500))
+       and method == "POST" and opts.body
+    then
+        local _chunk, _err
+        local reader = opts.body
+        repeat
+            _chunk, _err = reader()
+        until not _chunk
+    end
+
+    return res, hp, err
+end
+
+
+function _M.flush_response(hp, res, buffer_num, buffer_size, calc_timeout)
+    local buffer_num = buffer_num or 1
+    local buffer_size = buffer_size or 8192
+    local buffer = {}
+
+    local chunk, err
+    local reader = res.body_reader
+    repeat
+        if is_func(calc_timeout) then
+            hp.read_timeout = (calc_timeout(reqlimit.STATE_READ) or 0) * 1000
+        end
+
+        chunk, err = reader(buffer_size)
+
+        if err then
+            err = tostring(err) or "error"
+            if err == "timeout" then
+                err = "read timeout"
+            end
+            set_xstate(err)
+
+            if err ~= "closed" then
+                ngx.log(ngx.WARN, "failed to read response: ", err)
+            else
+                ngx.log(ngx.INFO, "failed to read response: ", err)
+            end
+
+            if err == "exceeds maxsize" then
+                _M.close(hp)
+                if not ngx.headers_sent then
+                    set_uptime()
+                    return ngx.exit(ngx.HTTP_FORBIDDEN)
+                end
+            end
+
+            if not ngx.headers_sent then
+                if ngx.ctx.slice then
+                    -- just return 423 for slice request when upstream
+                    -- prematurely closed connection
+                    ngx.status = 423
+                    set_uptime()
+                    return ngx.exit(423)
+                end
+
+                local ok, err = send_headers()
+                if not ok then
+                    ngx.log(ngx.WARN, "failed to send headers: ", err)
+                end
+            end
+
+            ngx.flush(true) -- flush response headers to client forcibly
+            set_uptime()
+            return ngx.exit(444)
+        end
+
+        if chunk then
+            if buffer_num == 1 then
+                -- The buffer_num in Shanks is always 1 so far. So never
+                -- insert it into table for avoiding the overhead table.concat.
+                ngx.print(chunk)
+                ngx.flush(true)
+            else
+                insert(buffer, chunk)
+                if #buffer >= buffer_num then
+                    ngx.print(concat(buffer))
+                    ngx.flush(true)
+                    buffer = {}
+                end
+            end
+        end
+    until not chunk
+
+    if #buffer > 0 then
+        ngx.print(concat(buffer))
+        ngx.flush(true)
+    end
+
+    --get stream size from httpipe.total_size
+    if ngx.var.upsize == "-" and ngx.var.from_upyun ~= "1" then
+        ngx.var.upsize = hp and hp.total_size or 0
+    end
+
+    set_uptime()
+end
+
+
+function _M.raise_capture_error(err, raise_error_filter)
+    set_uptime()
+
+    err = tostring(err) or "error"
+    ngx.log(ngx.WARN, "failed to capture: ", err)
+
+    set_xstate(err)
+
+    if type(raise_error_filter) == "function" then
+        return raise_error_filter(err)
+    end
+
+    if err == "timeout" then
+        return ngx.exit(ngx.HTTP_GATEWAY_TIMEOUT)
+    end
+
+    return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)
+end
+
+
+function _M.exec(method, uri, source, opts)
+    local res, hp, err = _M.capture(method, uri, source, opts)
+
+    if not res then
+        return _M.raise_capture_error(err)
+    end
+
+    if type(res.status) == "number" then
+        for k, v in pairs(res.headers) do
+            ngx.header[k] = v
+        end
+    end
+
+    ngx.status = res.status
+
+    if not hp:eof() then
+        return _M.flush_response(hp, res)
+    else
+        set_uptime()
+        return ngx.exit(res.status)
+    end
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/modules/reqlimit.lua

@@ -0,0 +1,83 @@
+-- Copyright(C) 2016 Jingli Chen (Wine93), UPYUN Inc.
+
+local math  = require "math"
+local utils = require "modules.utils"
+
+local ngx            = ngx
+local tonumber       = tonumber
+local floor          = math.floor
+local min            = math.min
+local ngx_now        = ngx.now
+local req_start_time = ngx.req.start_time
+local get_method     = ngx.get_method
+local is_tab         = utils.is_tab
+
+local _M = {
+    _VERSION = "0.01",
+
+    STATE_REQUEST = 0, -- connect, send, read header
+    STATE_READ    = 1, -- read body
+
+    TIME_RUN_OUT = 10,
+}
+local TIME_WEIGHT = { 0.1, 0.2, 0.7 }
+local TIME_LIMIT = 0.001
+
+
+function _M.register_callback(config)
+    if not is_tab(config) then
+        config = {}
+    end
+
+    local conn_timeout = config.cdn_timeout or 30
+    local send_timeout = config.cdn_send_timeout or 60
+    local read_timeout = config.cdn_read_timeout or 60
+    local start_time = req_start_time()
+    local bytes_sent = 0
+
+    local function calc_timeout(phase, slice)
+        local elapsed = ngx_now() - start_time
+        local downstream_timeout
+
+        if slice then
+            downstream_timeout = config.cdn_downstream_timeout_slice or 60
+        else
+            downstream_timeout = config.cdn_downstream_timeout or 60
+        end
+
+        local useable_time = downstream_timeout - elapsed
+
+        if phase == _M.STATE_REQUEST then
+            if useable_time <= 0 then
+                return
+            end
+
+            local timeout = { conn_timeout, send_timeout, read_timeout }
+            local time_alloc = {}
+            for i = 1, 3 do
+                time_alloc[i] = useable_time * TIME_WEIGHT[i]
+                time_alloc[i] = min(time_alloc[i], timeout[i])
+
+                if time_alloc[i] < TIME_LIMIT then
+                    time_alloc[i] = nil
+                end
+            end
+
+            return time_alloc[1], time_alloc[2], time_alloc[3]
+        end
+
+        -- phase == STATE_READ
+        local time4read = read_timeout
+        if bytes_sent == 0 then
+            bytes_sent = tonumber(ngx.var.bytes_sent) or 0
+            time4read = min(useable_time, read_timeout)
+        end
+
+        return time4read
+    end
+
+    return calc_timeout
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/modules/utils.lua

@@ -0,0 +1,251 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+-- Copyright (c) Netease, xiaojing tang
+
+local base     = require "resty.core.base"
+local debug    = require "debug"
+local argutils = require "resty.argutils"
+
+local _M = {}
+
+local next  = next
+local sub   = string.sub
+local gsub  = string.gsub
+local len   = string.len
+local find  = string.find
+local lower = string.lower
+
+local registry     = debug.getregistry()
+local ngx_match    = ngx.re.match
+local table_insert = table.insert
+local set_uri_args = ngx.req.set_uri_args
+local delete_args  = argutils.delete_args
+local escape_uri   = ngx.escape_uri
+local tab_insert   = table.insert
+local tab_concat   = table.concat
+local set_header   = ngx.req.set_header
+
+_M.is_str = function(s) return type(s) == "string" end
+_M.is_num = function(n) return type(n) == "number" end
+_M.is_tab = function(t) return type(t) == "table" end
+_M.is_func = function(f) return type(f) == "function" end
+_M.null = function(e) return e == nil or e == ngx.null end
+
+
+local inner_addr = {
+    ["10"]  = {0, 255},    -- 10.0.0.0-10.255.255.255
+    ["172"] = {16, 31},    -- 172.16.0.0-172.31.255.255
+    ["192"] = {168, 168},  -- 192.168.0.0-192.168.255.255
+}
+
+
+function _M.unormalize_header(s)
+    return gsub(lower(s), "-", "_")
+end
+
+
+function _M.is_empty_table(t)
+    return next( t ) == nil
+end
+
+
+function _M.escape_path(path)
+    return gsub(path, "([^/]+)", function (s) return escape_uri(s) end)
+end
+
+
+function _M.table_empty(t)
+    return next(t) == nil
+end
+
+
+function _M.table_in(t, key)
+    for k, v in ipairs(t) do
+        if key == v then
+            return true
+        end
+    end
+
+    return false
+end
+
+
+function _M.get_ip_port(host)
+    if not host then
+        return nil, nil
+    end
+
+    local pos = find(host, ":", 1)
+    if not pos then
+        return host, 80
+    end
+
+    local ip = sub(host, 1, pos - 1)
+    local port = tonumber(sub(host, pos + 1))
+
+    return ip, port
+end
+
+
+function _M.is_ip_format(host)
+    if not host == nil then
+        return false
+    end
+
+    local pos = find(host, ":", 1)
+    if pos then
+        host = sub(host, 1, pos - 1)
+    end
+
+    local m, err = ngx_match(host, [[^\d+\.\d+\.\d+\.\d+$]], "ijso")
+    if m and m[0] then
+        return true
+    else
+        return false
+    end
+end
+
+
+function _M.log_request_headers(msg)
+    local h = ngx.req.get_headers()
+    local t = {}
+    for k, v in pairs(h) do
+        tab_insert(t, ("%s=%s"):format(k, v))
+    end
+    ngx.log(ngx.ERR, msg, ", request headers:", tab_concat(t, " "))
+end
+
+
+function _M.gen_resource(bucket, object)
+    local resource = ""
+    if bucket and bucket ~= "" then
+        resource = resource .. "/" .. bucket
+    end
+
+    if object and object ~= "" then
+        resource = resource .. "/" .. object
+    end
+
+    return resource
+end
+
+
+function _M.split_string(s, p)
+    local res = {}
+    gsub(s, '[^'..p..']+', function(w) table_insert(res, w) end)
+    return res
+end
+
+
+function _M.is_inner_ip(ip)
+    local sub = _M.split_string(ip, ".")
+
+    if sub and #sub == 4 and sub[1] and sub[2] then
+        local range = inner_addr[sub[1]]
+        local int_sub2 = tonumber(sub[2]) or 0
+        if range and int_sub2 >= range[1] and int_sub2 <= range[2] then
+            return true
+        end
+    end
+
+    return false
+end
+
+
+function _M.set_isp(ip, special_ip)
+    -- from multi line, do not set isp
+    local isp = ngx.var.http_x_nos_isp
+    if isp then
+        set_header("x-nos-isp", nil)
+        set_header("x-inner-isp", isp)
+        return
+    end
+
+    if _M.is_inner_ip(ip) then
+        set_header("x-inner-isp", "inner")
+    elseif special_ip and special_ip[ip] then
+        set_header("x-inner-isp", "bp")
+    else
+        set_header("x-inner-isp", "outer")
+    end
+end
+
+
+function _M.delete_uri_args(del_args)
+    local args = ngx.var.args
+    if not args or not del_args then
+        return
+    end
+
+    local opts = { delete_bool_value = true }
+    for _, k in ipairs(del_args) do
+        args = delete_args(args, "^" .. k .. "$", ".*", opts)
+    end
+
+    set_uri_args(args)
+end
+
+
+function _M.table_dup(ori_tab)
+    if type(ori_tab) ~= "table" then
+        return ori_tab
+    end
+    local new_tab = {}
+    for k, v in pairs(ori_tab) do
+        if type(v) == "table" then
+            new_tab[k] = _M.table_dup(v)
+        else
+            new_tab[k] = v
+        end
+    end
+    return new_tab
+end
+
+
+-- stash the old ngx.ctx, create a new anchor.
+-- Note: stash_ngx_ctx and apply_ngx_ctx need to be called in pairs, otherwise
+-- the overhead memory leak will happens!
+function _M.stash_ngx_ctx()
+    local ctxs = registry.ngx_lua_ctx_tables
+    local ctx_ref = base.ref_in_table(ctxs, ngx.ctx)
+    ngx.var.ctx_ref = tostring(ctx_ref)
+end
+
+
+-- restore the old ngx.ctx with the anchor fetched from stash_ngx_ctx, replace
+-- current the ngx.ctx, you need to call it early after Nginx internal redirect
+-- happens.
+-- Note: stash_ngx_ctx and apply_ngx_ctx need to be called in pairs, otherwise
+-- the overhead memory leak will happens!
+function _M.apply_ngx_ctx()
+    local ctx_ref = tonumber(ngx.var.ctx_ref)
+    if not ctx_ref then
+        return
+    end
+
+    local ctxs = registry.ngx_lua_ctx_tables
+    local origin_ngx_ctx = ctxs[ctx_ref]
+    ngx.ctx = origin_ngx_ctx
+
+    --- FIXME unref the ctx_ref for avoiding memory leak
+    local FREE_LIST_REF = 0
+    ctxs[ctx_ref] = ctxs[FREE_LIST_REF]
+    ctxs[FREE_LIST_REF] = ctx_ref
+    ngx.var.ctx_ref = ""
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/snapshot.lua

@@ -0,0 +1,18 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+local httproxy = require "modules.httproxy"
+
+return httproxy.exec(ngx.var.request_method, ngx.var.request_uri, "snapshot");

curve-snapshotcloneserver-nginx/conf/mime.types

@@ -0,0 +1,94 @@
+types {
+    text/html                                        html htm shtml;
+    text/css                                         css;
+    text/xml                                         xml;
+    image/gif                                        gif;
+    image/jpeg                                       jpeg jpg;
+    application/javascript                           js;
+    application/atom+xml                             atom;
+    application/rss+xml                              rss;
+
+    text/mathml                                      mml;
+    text/plain                                       txt;
+    text/vnd.sun.j2me.app-descriptor                 jad;
+    text/vnd.wap.wml                                 wml;
+    text/x-component                                 htc;
+
+    image/png                                        png;
+    image/svg+xml                                    svg svgz;
+    image/tiff                                       tif tiff;
+    image/vnd.wap.wbmp                               wbmp;
+    image/webp                                       webp;
+    image/x-icon                                     ico;
+    image/x-jng                                      jng;
+    image/x-ms-bmp                                   bmp;
+
+    application/font-woff                            woff;
+    application/java-archive                         jar war ear;
+    application/json                                 json;
+    application/mac-binhex40                         hqx;
+    application/msword                               doc;
+    application/pdf                                  pdf;
+    application/postscript                           ps eps ai;
+    application/rtf                                  rtf;
+    application/vnd.apple.mpegurl                    m3u8;
+    application/vnd.google-earth.kml+xml             kml;
+    application/vnd.google-earth.kmz                 kmz;
+    application/vnd.ms-excel                         xls;
+    application/vnd.ms-fontobject                    eot;
+    application/vnd.ms-powerpoint                    ppt;
+    application/vnd.oasis.opendocument.graphics      odg;
+    application/vnd.oasis.opendocument.presentation  odp;
+    application/vnd.oasis.opendocument.spreadsheet   ods;
+    application/vnd.oasis.opendocument.text          odt;
+    application/vnd.openxmlformats-officedocument.presentationml.presentation
+                                                     pptx;
+    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+                                                     xlsx;
+    application/vnd.openxmlformats-officedocument.wordprocessingml.document
+                                                     docx;
+    application/vnd.wap.wmlc                         wmlc;
+    application/x-7z-compressed                      7z;
+    application/x-cocoa                              cco;
+    application/x-java-archive-diff                  jardiff;
+    application/x-java-jnlp-file                     jnlp;
+    application/x-makeself                           run;
+    application/x-perl                               pl pm;
+    application/x-pilot                              prc pdb;
+    application/x-rar-compressed                     rar;
+    application/x-redhat-package-manager             rpm;
+    application/x-sea                                sea;
+    application/x-shockwave-flash                    swf;
+    application/x-stuffit                            sit;
+    application/x-tcl                                tcl tk;
+    application/x-x509-ca-cert                       der pem crt;
+    application/x-xpinstall                          xpi;
+    application/xhtml+xml                            xhtml;
+    application/xspf+xml                             xspf;
+    application/zip                                  zip;
+
+    application/octet-stream                         bin exe dll;
+    application/octet-stream                         deb;
+    application/octet-stream                         dmg;
+    application/octet-stream                         iso img;
+    application/octet-stream                         msi msp msm;
+
+    audio/midi                                       mid midi kar;
+    audio/mpeg                                       mp3;
+    audio/ogg                                        ogg;
+    audio/x-m4a                                      m4a;
+    audio/x-realaudio                                ra;
+
+    video/3gpp                                       3gpp 3gp;
+    video/mp2t                                       ts;
+    video/mp4                                        mp4;
+    video/mpeg                                       mpeg mpg;
+    video/quicktime                                  mov;
+    video/webm                                       webm;
+    video/x-flv                                      flv;
+    video/x-m4v                                      m4v;
+    video/x-mng                                      mng;
+    video/x-ms-asf                                   asx asf;
+    video/x-ms-wmv                                   wmv;
+    video/x-msvideo                                  avi;
+}

curve-snapshotcloneserver-nginx/conf/nginx.conf

@@ -0,0 +1,83 @@
+# -*- mode: nginx -*-
+# vim: set expandtab tabstop=4 shiftwidth=4:
+
+#user  nobody;
+worker_processes  5;
+error_log  logs/error.log  info;
+pid        logs/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for", "$http_range", "$request_time" '
+                      '"$upstream_addr" "$upnode" "$upcode" ""$uptime" ';
+
+    access_log  logs/access.log  main;
+
+    sendfile on;
+    keepalive_timeout  65;
+
+    send_timeout 60;
+    proxy_read_timeout 60;
+    proxy_send_timeout 60;
+    proxy_connect_timeout 1;
+    proxy_max_temp_file_size 0m;
+
+    client_header_buffer_size 4k;
+    large_client_header_buffers 8 16k;
+    client_body_buffer_size 128k;
+    client_max_body_size 200m;
+
+    port_in_redirect off;
+    open_log_file_cache max=2048 inactive=60s min_uses=2 valid=15m;
+
+    resolver 119.29.29.29 223.5.5.5 valid=1s;
+    resolver_timeout 5s;
+
+    lua_max_running_timers 512;
+    lua_max_pending_timers 1024;
+
+    lua_package_path "$prefix/lualib/?.lua;$prefix/app/lib/?.lua;$prefix/app/etc/?.lua;$prefix/app/src/?.lua;;";
+    lua_package_cpath "$prefix/lualib/?.so;/$prefix/app/lib/?.so;;";
+
+    lua_check_client_abort on;
+    lua_socket_log_errors off;
+    lua_http10_buffering off;
+
+    lua_shared_dict state 10m;
+    lua_shared_dict mutex 1m;
+    lua_shared_dict locks 1m;
+    lua_shared_dict config 1m;
+    lua_shared_dict cache 1m;
+
+    init_by_lua_file app/src/init.lua;
+    init_worker_by_lua_file app/src/init_worker.lua;
+
+
+    server {
+        listen 80 default_server;
+        server_name _;
+
+        set $upnode "-";
+        set $upcode "-";
+        set $upsize "-";
+        set $uptime "-";
+        set $uptimes "-";
+        set $upstats "-";
+        set $xstate "-";
+
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+
+        location / {
+            content_by_lua_file app/src/snapshot.lua;
+        }
+    }
+}

curve-snapshotcloneserver-nginx/docker/Dockerfile

@@ -0,0 +1,62 @@
+# Dockerfile - Debian 10 Buster - DEB version
+# https://github.com/openresty/docker-openresty
+
+ARG RESTY_IMAGE_BASE="debian"
+ARG RESTY_IMAGE_TAG="buster-slim"
+
+FROM ${RESTY_IMAGE_BASE}:${RESTY_IMAGE_TAG}
+
+LABEL maintainer="Evan Wies <evan@neomantra.net>"
+
+# RESTY_DEB_FLAVOR build argument is used to select other
+# OpenResty Debian package variants.
+# For example: "-debug" or "-valgrind"
+ARG RESTY_DEB_FLAVOR=""
+ARG RESTY_DEB_VERSION="=1.15.8.3-1~buster1"
+ARG RESTY_IMAGE_BASE="debian"
+ARG RESTY_IMAGE_TAG="buster-slim"
+
+LABEL resty_image_base="${RESTY_IMAGE_BASE}"
+LABEL resty_image_tag="${RESTY_IMAGE_TAG}"
+LABEL resty_deb_flavor="${RESTY_DEB_FLAVOR}"
+LABEL resty_deb_version="${RESTY_DEB_VERSION}"
+
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        ca-certificates \
+        gettext-base \
+        gnupg2 \
+        lsb-release \
+        software-properties-common \
+        wget \
+    && wget -qO /tmp/pubkey.gpg https://openresty.org/package/pubkey.gpg \
+    && DEBIAN_FRONTEND=noninteractive apt-key add /tmp/pubkey.gpg \
+    && rm /tmp/pubkey.gpg \
+    && DEBIAN_FRONTEND=noninteractive add-apt-repository -y "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" \
+    && DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \
+        gnupg2 \
+        lsb-release \
+        software-properties-common \
+        wget \
+    && DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        openresty${RESTY_DEB_FLAVOR}${RESTY_DEB_VERSION} \
+    && DEBIAN_FRONTEND=noninteractive apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /var/run/openresty \
+    && ln -sf /dev/stdout /usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/logs/access.log \
+    && ln -sf /dev/stderr /usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/logs/error.log
+
+# Add additional binaries into PATH for convenience
+ENV PATH="$PATH:/usr/local/openresty${RESTY_DEB_FLAVOR}/luajit/bin:/usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/sbin:/usr/local/openresty${RESTY_DEB_FLAVOR}/bin"
+
+# Copy nginx configuration files
+COPY nginx.conf /usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/conf/nginx.conf
+COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf
+
+CMD ["/usr/bin/openresty", "-g", "daemon off;"]
+
+# Use SIGQUIT instead of default SIGTERM to cleanly drain requests
+# See https://github.com/openresty/docker-openresty/blob/master/README.md#tips--pitfalls
+STOPSIGNAL SIGQUIT

curvefs/Makefile

@@ -0,0 +1,98 @@
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+.PHONY: build install deploy core config topo start stop reload status clean mount umount
+
+prefix?= "$(PWD)/devops/projects"
+release?= 0
+only?= "*"
+hosts?= "*"
+tag?= "curvefs:unknown"
+
+define deploy_begin
+	@bash util/deploy.sh begin
+endef
+
+define deploy_end
+	@bash util/deploy.sh end
+endef
+
+define only_service
+	$(call deploy_begin)
+	@bash util/deploy.sh --hosts=$(hosts) --only="etcd" --tags=$(1)
+	@bash util/deploy.sh --hosts=$(hosts) --only="mds" --tags=$(1)
+	@bash util/deploy.sh --hosts=$(hosts) --only="metaserver" --tags=$(1)
+	$(call deploy_end)
+endef
+
+define only_specify
+	$(call deploy_begin)
+	@bash util/deploy.sh --hosts=$(hosts) --only=$(1) --tags=$(2)
+	$(call deploy_end)
+endef
+
+build:
+	@bash util/build.sh --only=$(only) --release=$(release)
+
+install:
+	@bash util/install.sh --prefix=$(prefix) --only=$(only)
+
+image:
+	@bash util/image.sh $(tag)
+
+deploy:
+	$(call deploy_begin)
+	@bash util/deploy.sh --hosts="etcd" --only="etcd" --tags="core,config,start"
+	@bash util/deploy.sh --hosts="mds" --only="mds" --tags="core,config,start"
+	@bash util/deploy.sh --hosts="metaserver" --only="metaserver" --tags="core,config,start"
+	@bash util/deploy.sh --hosts="tools" --only="tools" --tags="core,config,topo"
+	@bash util/deploy.sh --hosts="client" --only="client" --tags="core,config"
+	$(call deploy_end)
+
+core:
+	$(call only_specify,$(only),core)
+
+config:
+	$(call only_specify,$(only),config)
+
+topo:
+	$(call only_specify,tools,topo)
+
+debug:
+	$(call only_specify,$(only),debug)
+
+start:
+ifeq ($(only), "*")
+	$(call only_service,start)
+else
+	$(call only_specify,$(only),start)
+endif
+
+stop:
+ifeq ($(only), "*")
+	$(call only_service,stop)
+else
+	$(call only_specify,$(only),stop)
+endif
+
+reload:
+ifeq ($(only), "*")
+	$(call only_service,restart)
+else
+	$(call only_specify,$(only),restart)
+endif
+
+status:
+ifeq ($(only), "*")
+	$(call only_service,status)
+else
+	$(call only_specify,$(only),status)
+endif
+
+clean:
+	$(call only_specify,$(only),clean)
+
+mount:
+	$(call only_specify,client,mount)
+
+umount:
+	$(call only_specify,client,umount)

curvefs/README.md

@@ -3,16 +3,127 @@ CURVEFS
 
 Curve FileSystem
 
+Table of Contents
+===
+* [Requirement](#requirement)
+* [Quick Start](#quick-start)
+* [Devops](#devops)
+* [Hint](#hint)
+
+Requirement
+===
+
+* bazel
+* ansible
+* linux*
+
+[Back to Toc](#table-of-contents)
+
 Quick Start
 ===
 
-```
-$ make build dep=1
-```
+NOTE: If you are using CentOS-8, you can refer to this issue to compile CurveFS: https://github.com/opencurve/curve/issues/807
 
-OR
+step 1: clone repository, run build and install:
 
 ```
-$ make dep
+$ git clone https://github.com/opencurve/curve.git
+$ cd curve/curvefs
 $ make build
+$ make install
+$ make install only=etcd
 ```
+
+step 2: edit ansible config file, inventory file and client config file:
+
+* devops/ansible.cfg:
+    * `remoter_user`: we use this user to log in to the remote machine, create file and start service
+    * `private_key_file`: paste `remote_user`'s private key to `devops/ssh/pub_rsa` (you can also save the private key anywhere locally and point the `private_key_file` to it)
+    * Please make sure that the mode of private key file which `private_key_file` specfied is `600` (`-rw-------`)
+    * Please make sure that the public key already exists in `remote_user`'s ssh file (`~/.ssh/authorized_keys`)
+* inventory/server.ini:
+    * You can specify which hosts to deploy which services
+    * Please specify the client mount path (`client_mount_path`), the path will create automatic if it not exist
+    * Please specify the filesystem name which client mount (`client_mount_fsname`), the filesystem will create automatic if it not exist
+    * The `tools` only used to create topology, you can select one of `mds` hosts
+* conf/{client.conf, metaserver.conf}
+    * `s3.ak`
+    * `s3.sk`
+    * `s3.endpoint`
+    * `s3.bucket_name`
+
+step 3: deploy all and mount curve filesystem:
+
+```
+$ make deploy
+$ make mount
+```
+
+once this is done, you can enter the mount path and do anything like local filesystem.
+
+[Back to Toc](#table-of-contents)
+
+Devops
+===
+
+| command                                  | description                                   |
+| :---                                     | :---                                          |
+| make build [only=ONLY] [release=1]       | compile                                       |
+| make install [only=ONLY] [prefix=PREFIX] | install                                       |
+| [sudo] make image tag=TAG                | make docker image (maybe need sudo privilege) |
+| make deploy                              | deploy all                                    |
+| make core [only=ONLY] [hosts=HOSTS]      | sync binary file                              |
+| make config [only=ONLY] [hosts=HOSTS]    | sync config file                              |
+| make start [only=ONLY] [hosts=HOSTS]     | start service                                 |
+| make stop [only=ONLY] [hosts=HOSTS]      | stop service                                  |
+| make reload [only=ONLY] [hosts=HOSTS]    | restart service                               |
+| make status [only=ONLY] [hosts=HOSTS]    | show service status                           |
+| make clean [only=ONLY] [hosts=HOSTS]     | clean environment (include all created files) |
+| make topo                                | create topology                               |
+| make mount                               | mount curve filesystem                        |
+| make umount                              | umount curve filesystem                       |
+
+
+* If you want to execute action for specfied service, you can use `only` option, e.g: `make start only=mds`
+* If you want to execute action in specfied host, you can use `hosts` option, e.g: `make start hosts=machine1`
+* You can also specify both `only` and `hosts` option, e.g: `make start only=mds hosts=machine1:machine2`
+* The `only` option can be one of the following values: `etcd`、`mds`、`metaserver`、`space`、`client`、 `tools`
+
+[Back to Toc](#table-of-contents)
+
+HINT
+===
+
+* The default install prefix of projects is `devops/projects`
+* If you want to modify the service's config, you can modify the config under `conf` directory, then run `make config`
+* You can use `make status` to show service status, include active status, listen address, memory usage and etc:
+
+```
+$ make status only=mds
+
+curve-vm1:
+mds.service - CurveFS Mds
+    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
+  Main PID: 326373 (curvefs_mds)
+    Daemon: True
+    Listen: 0.0.0.0:26700, 0.0.0.0:27700
+       Mem: 51688 KB
+
+curve-vm2:
+mds.service - CurveFS Mds
+    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
+  Main PID: 326381 (curvefs_mds)
+    Daemon: True
+    Listen: 0.0.0.0:37700
+       Mem: 40688 KB
+
+curve-vm3:
+mds.service - CurveFS Mds
+    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
+  Main PID: 326401 (curvefs_mds)
+    Daemon: True
+    Listen: 0.0.0.0:17700
+       Mem: 55520 KB
+```
+
+[Back to Toc](#table-of-contents)

curvefs/conf/client.conf

@@ -17,14 +17,6 @@ mdsOpt.rpcRetryOpt.normalRetryTimesBeforeTriggerWait=3
 mdsOpt.rpcRetryOpt.waitSleepMs=1000
 mdsOpt.rpcRetryOpt.addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702  # __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_port") }} __ANSIBLE_TEMPLATE__
 
-
-#
-# lease options
-#
-# client and mds lease time, default is 20s
-mds.leaseTimesUs=20000000
-mds.refreshTimesPerLease=5
-
 #### metaCacheOpt
 # Gets the number of retries for the leader
 metaCacheOpt.metacacheGetLeaderRetry=3
@@ -33,38 +25,31 @@ metaCacheOpt.metacacheRPCRetryIntervalUS=100000
 # RPC timeout of get leader
 metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
 
-#### executorOpt
-# executorOpt rpc with metaserver
+#### excutorOpt
+# excutorOpt rpc with metaserver
 # rpc retry times with metaserver
-executorOpt.maxRetry=4294967295
-# internal rpc retry times with metaserver
-executorOpt.maxInternalRetry=3
+excutorOpt.maxRetry=1000000
 # Retry sleep time between failed RPCs
-executorOpt.retryIntervalUS=100000
+excutorOpt.retryIntervalUS=500
 # RPC timeout for communicating with metaserver
-executorOpt.rpcTimeoutMS=1000
-# RPC stream idle timeout
-executorOpt.rpcStreamIdleTimeoutMS=500
+excutorOpt.rpcTimeoutMS=1000
 # The maximum timeout RPC time of the retry request.
 # The timeout time will follow the exponential backoff policy.
 # Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
-executorOpt.maxRPCTimeoutMS=8000
+excutorOpt.maxRPCTimeoutMS=8000
 # Maximum sleep time between retry requests.
 # when the network is congested or the metaserver is overloaded,
 # it is necessary to increase the sleep time
-executorOpt.maxRetrySleepIntervalUS=8000000
-executorOpt.minRetryTimesForceTimeoutBackoff=5
-executorOpt.maxRetryTimesBeforeConsiderSuspend=20
-# batch limit of get inode attr and xattr
-executorOpt.batchInodeAttrLimit=10000
+excutorOpt.maxRetrySleepIntervalUS=8000000
+excutorOpt.minRetryTimesForceTimeoutBackoff=5
+excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
 
 #### spaceserver
-spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-spaceServer.rpcTimeoutMs=1000
+spaceserver.spaceaddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
+spaceserver.rpcTimeoutMs=1000
 
 #### bdev
-# curve client's config file
-bdev.confPath=/etc/curve/client.conf
+bdev.confpath=/etc/curve/client.conf
 
 #### extentManager
 extentManager.preAllocSize=65536
@@ -77,142 +62,60 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
+fuseClient.attrTimeOut=1.0
+fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
-fuseClient.downloadMaxRetryTimes=3
-
-### kvcache opt
-fuseClient.supportKVcache=false
-fuseClient.setThreadPool=4
-fuseClient.getThreadPool=4
-
-# you shoudle enable it when mount one filesystem to multi mountpoints,
-# it gurantee the consistent of file after rename, otherwise you should
-# disable it for performance.
-fuseClient.enableMultiMountPointRename=true
-# splice will bring higher performance in some cases
-# but there might be a kernel issue that will cause kernel panic when enabling it
-# see https://lore.kernel.org/all/CAAmZXrsGg2xsP1CK+cbuEMumtrqdvD-NKnWzhNcvn71RV3c1yw@mail.gmail.com/
-# until this issue has been fixed, splice should be disabled
-fuseClient.enableSplice=false
-# thread number of listDentry when get summary xattr
-fuseClient.listDentryThreads=10
-# disable xattr on one mountpoint can fast 'ls -l'
-fuseClient.disableXattr=false
-# default data（s3ChunkInfo/volumeExtent） size in inode, if exceed will eliminate and try to get the merged one
-fuseClient.maxDataSize=1024
-# default refresh data interval 30s
-fuseClient.refreshDataIntervalSec=30
-fuseClient.warmupThreadsNum=10
-
-# the write throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgWriteBytes=0
-# the write burst bps of fuseClient, default no limit
-fuseClient.throttle.burstWriteBytes=0
-# the times that write burst bps can continue, default 180s
-fuseClient.throttle.burstWriteBytesSecs=180
-
-# the write throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgWriteIops=0
-# the write burst iops of fuseClient, default no limit
-fuseClient.throttle.burstWriteIops=0
-# the times that write burst Iops can continue, default 180s
-fuseClient.throttle.burstWriteIopsSecs=180
-
-# the read throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgReadBytes=0
-# the read burst bps of fuseClient, default no limit
-fuseClient.throttle.burstReadBytes=0
-# the times that read burst bps can continue, default 180s
-fuseClient.throttle.burstReadBytesSecs=180
-
-# the read throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgReadIops=0
-# the read burst Iops of fuseClient, default no limit
-fuseClient.throttle.burstReadIops=0
-# the times that read burst Iops can continue, default 180s
-fuseClient.throttle.burstReadIopsSecs=180
-
-#### filesystem metadata
-# {
-# fs.disableXattr:
-#   if you want to get better metadata performance,
-#   you can mount fs with |fs.disableXattr| is true
-#
-# fs.lookupCache.negativeTimeoutSec:
-#   entry which not found will be cached if |timeout| > 0
-fs.cto=true
-fs.maxNameLength=255
-fs.disableXattr=false
-fs.accessLogging=true
-fs.kernelCache.attrTimeoutSec=3600
-fs.kernelCache.dirAttrTimeoutSec=3600
-fs.kernelCache.entryTimeoutSec=3600
-fs.kernelCache.dirEntryTimeoutSec=3600
-fs.lookupCache.negativeTimeoutSec=0
-fs.lookupCache.minUses=1
-fs.lookupCache.lruSize=100000
-fs.dirCache.lruSize=5000000
-fs.openFile.lruSize=65536
-fs.attrWatcher.lruSize=5000000
-fs.rpc.listDentryLimit=65536
-fs.deferSync.delay=3
-fs.deferSync.deferDirMtime=false
-# }
+fuseClient.flushPeriodSec=5
+fuseClient.maxNameLength=255
+fuseClient.iCacheLruSize=65536
+fuseClient.dCacheLruSize=65536
+fuseClient.enableICacheMetrics=true
+fuseClient.enableDCacheMetrics=true
 
 #### volume
 volume.bigFileSize=1048576
 volume.volBlockSize=4096
 volume.fsBlockSize=4096
 
-# allocator type, supported {bitmap}
-volume.allocator.type=bitmap
-
-## for bitmap allocator
-# size of each bit, default is 4MiB
-volume.bitmapAllocator.sizePerBit=4194304
-
-# small allocation proportion [0-1]
-volume.bitmapAllocator.smallAllocProportion=0.2
-
-# number of block groups that allocated once
-volume.blockGroup.allocateOnce=4
-
 #### s3
-# this is for test. if s3.fakeS3=true, all data will be discarded
-s3.fakeS3=false
-s3.pageSize=65536
+s3.blocksize=4194304
+s3.chunksize=67108864
+# the max size that fuse send
+s3.fuseMaxSize=131072
+s3.pagesize=65536
 # prefetch blocks that disk cache use
 s3.prefetchBlocks=1
 # prefetch threads
 s3.prefetchExecQueueNum=1
 # start sleep when mem cache use ratio is greater than nearfullRatio,
-# sleep time increase follow with mem cache use ratio, baseSleepUs is baseline.
+# sleep time increase follow with mem cache use raito, baseSleepUs is baseline.
 s3.nearfullRatio=70
 s3.baseSleepUs=500
 
+# TODO(huyao): use more meaningfull name
 # background thread schedule time
-s3.threadScheduleInterval=3
+s3.intervalSec=3
 # data cache flush wait time
-s3.cacheFlushIntervalSec=5
-# write cache < 8,388,608 (8MB) is not allowed
+s3.flushIntervalSec=5
 s3.writeCacheMaxByte=838860800
 s3.readCacheMaxByte=209715200
-# file cache read thread num
-s3.readCacheThreads=5
+s3.dataCrc=true
+s3.endpoint=
+s3.bucket_name=
+s3.ak=
+s3.sk=
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False
-s3.region=us-east-1
-s3.maxConnections=500
-s3.connectTimeout=60000
-s3.requestTimeout=10000
+s3.max_connections=32
+s3.connect_timeout=60000
+s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel=4
+s3.loglevel=4
 s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
-s3.asyncThreadNum=500
+s3.async_thread_num=30
 # limit all inflight async requests' bytes, |0| means not limited
-s3.maxAsyncRequestInflightBytes=104857600
-s3.chunkFlushThreads=5
+s3.max_async_request_inflight_bytes=104857600
 # throttle
 s3.throttle.iopsTotalLimit=0
 s3.throttle.iopsReadLimit=0
@@ -220,17 +123,11 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
-s3.useVirtualAddressing=false
-# The interval between read failures and retries will become larger and larger,
-# and when the max is reached, retry will be performed at a fixed time.
-s3.maxReadRetryIntervalMs = 1000
-# retry interval
-s3.readRetryIntervalMs = 100
+
+
 # TODO(hongsong): limit bytes、iops/bps
 #### disk cache options
-# 0:not enable disk cache
-# 1:readonly
-# 2:read/write
+# 0:not enable disk cache 1:onlyread 2:read/write
 diskCache.diskCacheType=2  # __ANSIBLE_TEMPLATE__ {{ client_disk_cache_type | default('2') }} __ANSIBLE_TEMPLATE__
 # the file system writes files use flush or not
 diskCache.forceFlush=true
@@ -242,26 +139,23 @@ diskCache.asyncLoadPeriodMs=5
 # util less than safeRatio
 diskCache.fullRatio=90
 diskCache.safeRatio=70
-diskCache.threads=5
 # the max size disk cache can use
 diskCache.maxUsableSpaceBytes=107374182400
-# the max files that can cache
-diskCache.maxFileNums=1000000
 # the max time system command can run
 diskCache.cmdTimeoutSec=300
 # directory of disk cache
 diskCache.cacheDir=/mnt/curvefs_cache  # __CURVEADM_TEMPLATE__ /curvefs/client/data/cache __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /mnt/curvefs_disk_cache/{{ 99999999 | random | to_uuid | upper }} __ANSIBLE_TEMPLATE__
 
-# the write throttle bps of disk cache, default no limit
-diskCache.avgFlushBytes=0
-# the write burst bps of disk cache, default no limit
-diskCache.burstFlushBytes=0
+# the write throttle bps of disk cache, default 80MB/s
+diskCache.avgFlushBytes=83886080
+# the write burst bps of disk cache, default 100MB/s
+diskCache.burstFlushBytes=104857600
 # the times that write burst bps can continue, default 180s
 diskCache.burstSecs=180
 # the write throttle iops of disk cache, default no limit
 diskCache.avgFlushIops=0
-# the read throttle bps of disk cache, default no limit
-diskCache.avgReadFileBytes=0
+# the read throttle bps of disk cache, default 80MB/s
+diskCache.avgReadFileBytes=83886080
 # the read throttle iops of disk cache, default no limit
 diskCache.avgReadFileIops=0
 
@@ -270,4 +164,4 @@ client.common.logDir=/data/logs/curvefs  # __CURVEADM_TEMPLATE__ /curvefs/client
 # we have loglevel: {0,3,6,9}
 # as the number increases, it becomes more and more detailed
 client.loglevel=0
-client.dummyServer.startPort=9000
+client.dummyserver.startport=9000

curvefs/conf/mds.conf

@@ -5,11 +5,6 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 # dummy server port
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
-mds.loglevel=0
-# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-mds.server.idleTimeoutSec=-1
-
 #
 # space options
 #
@@ -47,14 +42,16 @@ leader.electionTimeoutMs=0
 #
 # time interval flush data to db
 mds.topology.TopologyUpdateToRepoSec=60
-# max partition number in copyset 2^7
-mds.topology.MaxPartitionNumberInCopyset=128
-# inode number in each partition 2^20 [0, 2^20-1]
-mds.topology.IdNumberInPartition=1048576
-# default create partition number 12
-mds.topology.CreatePartitionNumber=12
-# max copyset num in metaserver
-mds.topology.MaxCopysetNumInMetaserver=100
+# the policy of choose pool 0:Random, 1:Weight
+mds.topology.ChoosePoolPolicy=0
+# partition number in each copyset 2^8
+mds.topology.PartitionNumberInCopyset=256
+# id number in each partition 2^24 [0, 2^24-1]
+mds.topology.IdNumberInPartition=16777216
+# create copyset number at a time
+mds.topology.CreateCopysetNumber=10
+# default create partition number 3
+mds.topology.CreatePartitionNumber=3
 # Topology update metric interval
 mds.topology.UpdateMetricIntervalSec=60
 
@@ -76,20 +73,8 @@ mds.heartbeat.clean_follower_afterMs=1200000
 #
 # recoverScheduler switch
 mds.enable.recover.scheduler=true
-# copysetScheduler switch
-mds.enable.copyset.scheduler=true
-# leaderScheduler switch
-mds.enable.leader.scheduler=true
 # RecoverScheduler round interval, the unit is second
 mds.recover.scheduler.intervalSec=5
-# copysetScheduler round interval, the unit is second
-mds.copyset.scheduler.intervalSec=5
-# leaderScheduler round interval, the unit is second
-mds.leader.scheduler.intervalSec=5
-# the percentage difference for copysetScheduler to
-# determine whether resource balancing is required
-# based on the difference in resource usage percent, default is 15%
-mds.copyset.scheduler.balanceRatioPercent=15
 # Concurrency of operator on each metaserver
 mds.schduler.operator.concurrent=1
 # transfer leader timeout, after the timeout, mds removes the operator from the memory
@@ -109,56 +94,3 @@ mds.scheduler.metaserver.cooling.timeSec=1800
 # the backend thread check whether fs is able to delete,
 # check partition of deleting fs is deleting
 mds.fsmanager.backEndThreadRunInterSec=10
-# number of threads that load space info of volume
-mds.fsmanager.reloadSpaceConcurrency=10
-# the client timeout is 20s default, umount fs if timeout
-mds.fsmanager.client.timeoutSec=20
-
-#### s3
-# TODO(huyao): use more meaningfull name
-# http = 0, https = 1
-s3.http_scheme=0
-s3.verify_SSL=False
-s3.region=us-east-1
-s3.maxConnections=32
-s3.connectTimeout=60000
-s3.requestTimeout=10000
-# Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel=4
-s3.logPrefix=/data/logs/curvefs/aws_  # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
-s3.asyncThreadNum=30
-# limit all inflight async requests' bytes, |0| means not limited
-s3.maxAsyncRequestInflightBytes=104857600
-# throttle
-s3.throttle.iopsTotalLimit=0
-s3.throttle.iopsReadLimit=0
-s3.throttle.iopsWriteLimit=0
-s3.throttle.bpsTotalMB=0
-s3.throttle.bpsReadMB=0
-s3.throttle.bpsWriteMB=0
-s3.useVirtualAddressing=false
-
-# TTL(millisecond) for distributed lock
-dlock.ttl_ms=5000
-# lock try timeout(millisecond) for distributed lock
-dlock.try_timeout_ms=300
-# lock try interval(millisecond) for distributed lock
-dlock.try_interval_ms=30
-
-#### Options for interactive with curvebs MDS
-# RPC total retry time with MDS
-bs.mds.maxRetryMs=8000
-# RPC timeout for once communication with MDS
-bs.mds.rpcTimeoutMs=500
-# The maximum timeout of RPC communicating with MDS.
-# The timeout of exponential backoff cannot exceed this value
-bs.mds.maxRPCTimeoutMs=2000
-# RPC with mds needs to sleep for a period of time before each retry
-bs.mds.rpcRetryIntervalUs=50000
-# Switch if the number of consecutive retries on the current MDS exceeds the limit.
-# The number of failures includes the number of timeout retries
-bs.mds.maxFailedTimesBeforeChangeMDS=2
-# The normal retry times for trigger wait strategy
-bs.mds.normalRetryTimesBeforeTriggerWait=3
-# sleep interval in ms for wait
-bs.mds.waitSleepMs=1000

curvefs/conf/metaserver.conf

@@ -5,21 +5,26 @@ trash.scanPeriodSec=600
 trash.expiredAfterSec=604800
 
 # s3
+s3.blocksize=4194304
+s3.chunksize=67108864
 # if s3.enableDeleteObjects set True, batch size limit the object num of delete count per delete request
 s3.batchsize=100
 # if s3 sdk support batch delete objects, set True; other set False
 s3.enableDeleteObjects=False
+s3.endpoint=
+s3.bucket_name=
+s3.ak=
+s3.sk=
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False
-s3.region=us-east-1
-s3.maxConnections=32
-s3.connectTimeout=60000
-s3.requestTimeout=10000
+s3.max_connections=32
+s3.connect_timeout=60000
+s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel=4
+s3.loglevel=4
 s3.logPrefix=/tmp/curvefs/metaserver/aws_
-s3.asyncThreadNum=10
+s3.async_thread_num=10
 # throttle
 s3.throttle.iopsTotalLimit=0
 s3.throttle.iopsReadLimit=0
@@ -27,7 +32,6 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
-s3.useVirtualAddressing=false
 # s3 workqueue
 s3compactwq.enable=True
 s3compactwq.thread_num=2
@@ -57,8 +61,6 @@ metaserver.common.logDir=/tmp/curvefs/metaserver  # __CURVEADM_TEMPLATE__ ${pref
 # we have loglevel: {3,6,9}
 # as the number increases, it becomes more and more detailed
 metaserver.loglevel=0
-# metaserver meta file path, every metaserver need persist MetaServerMetadata on its own disk
-metaserver.meta_file_path=./0/metaserver.dat  # __CURVEADM_TEMPLATE__ ${prefix}/data/metaserver.dat __CURVEADM_TEMPLATE__
 
 # copyset data uri
 # all uri (data_uri/raft_log_uri/raft_meta_uri/raft_snapshot_uri/trash.uri) are ${protocol}://${path}
@@ -72,7 +74,7 @@ copyset.data_uri=local://./0/copysets  # __CURVEADM_TEMPLATE__ local://${prefix}
 # if value set to 1, means all copysets are loaded one by one it may cause a long start-up time
 # if value bigger than 1, means at most |load_concurrency| copysets are loaded parallelly
 # but larger value may cause higher cpu/memory/disk usgae
-copyset.load_concurrency=5
+copyset.load_concurrency=1
 
 # if the difference between the applied_index of the current replica and the
 # committed_index on the leader is less than |finishLoadMargin|, it's
@@ -128,25 +130,16 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-#
-# Concurrent apply queue
-### concurrent apply queue options for each copyset
-### concurrent apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
+### apply queue options for each copyset
+### apply queue is used to isolate raft threads, each worker has its own queue
+### whan a task can be applied it's been pushed into a corresponding worker queue by certain rules
+# number of apply queue workers for each, each worker will start a indepent thread
+applyqueue.worker_count=1
 
-# worker_count: number of apply queue workers for each, each worker will start a indepent thread
-# queue_depth: apply queue depth for each copyset
+# apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-# write apply queue workers count
-applyqueue.write_worker_count=3
-# write apply queue depth
-applyqueue.write_queue_depth=1
-# read apply queue workers count
-applyqueue.read_worker_count=2
-# read apply queue depth
-applyqueue.read_queue_depth=1
-
+applyqueue.queue_depth=1
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -154,17 +147,12 @@ applyqueue.read_queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
-# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-server.idleTimeoutSec=-1
-
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
-# braft default is True. Setting to false can greatly improve performance
-# but data maybe lost when all the duplicates are powered off at the same time
+# braft default is True. Setting to false can greatly improve performance.
 # We can select according to the specified scene.
-braft.raft_sync=False
+braft.raft_sync=True
 # Sync log meta, snapshot meta and raft meta
 # braft default is False
 braft.raft_sync_meta=True
@@ -199,124 +187,3 @@ mds.heartbeat_timeoutMs=1000
 partition.clean.scanPeriodSec=10
 # partition clean manager delete inode every inodeDeletePeriodMs
 partition.clean.inodeDeletePeriodMs=500
-
-##### mdsOpt
-# RPC total retry time with MDS
-mdsOpt.mdsMaxRetryMS=16000
-# The maximum timeout of RPC communicating with MDS.
-# The timeout of exponential backoff cannot exceed this value
-mdsOpt.rpcRetryOpt.maxRPCTimeoutMS=2000
-# RPC timeout for once communication with MDS
-mdsOpt.rpcRetryOpt.rpcTimeoutMs=500
-# RPC with mds needs to sleep for a period of time before each retry
-mdsOpt.rpcRetryOpt.rpcRetryIntervalUS=50000
-# Switch if the number of consecutive retries on the current MDS exceeds the limit.
-# The number of failures includes the number of timeout retries
-mdsOpt.rpcRetryOpt.maxFailedTimesBeforeChangeAddr=2
-# The normal retry times for trigger wait strategy
-mdsOpt.rpcRetryOpt.normalRetryTimesBeforeTriggerWait=3
-# Sleep interval for wait
-mdsOpt.rpcRetryOpt.waitSleepMs=1000
-mdsOpt.rpcRetryOpt.addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_port") }} __ANSIBLE_TEMPLATE__
-
-#
-# storage settings
-#
-# storage type, "memory" or "rocksdb"
-storage.type=rocksdb
-# metaserver max memory quota bytes (default: 30GB)
-storage.max_memory_quota_bytes=32212254720
-# metaserver max disk quota bytes (default: 2TB)
-storage.max_disk_quota_bytes=2199023255552
-# whether need to compress the value for memory storage (default: False)
-storage.memory.compression=False
-# rocksdb block cache(LRU) capacity (default: 8GB)
-storage.rocksdb.block_cache_capacity=8589934592
-# rocksdb writer buffer manager capacity (default: 6GB)
-storage.rocksdb.write_buffer_manager_capacity=6442450944
-# Control whether write buffer manager cost block cache
-# If true, the total memory usage by rocksdb is limited by `block_cache_capacity`
-storage.rocksdb.WBM_cost_block_cache=false
-# Maximum number of concurrent background jobs (compactions and flushes)
-storage.rocksdb.max_background_jobs=16
-# Maxinum number of threads to perform a compaction job by simultaneously (default: 4)
-storage.rocksdb.max_subcompactions=4
-# Number of files to trigger level-0 compaction (default: 1)
-storage.rocksdb.level0_file_num_compaction_trigger=1
-# Control maximum total data size for a level (default: 1GB)
-storage.rocksdb.max_bytes_for_level_base=1073741824
-# rocksdb column family's write_buffer_size
-# for store inode which exclude its s3chunkinfo list (unit: bytes, default: 64MB)
-storage.rocksdb.unordered_write_buffer_size=67108864
-# rocksdb column family's max_write_buffer_number
-# for store inode which exclude its s3chunkinfo list (default: 3)
-storage.rocksdb.unordered_max_write_buffer_number=3
-# rocksdb column family's write_buffer_size
-# for store dentry and inode's s3chunkinfo list (unit: bytes, default: 128MB)
-storage.rocksdb.ordered_write_buffer_size=67108864
-# rocksdb column family's max_write_buffer_number
-# for store dentry and inode's s3chunkinfo list (default: 3)
-storage.rocksdb.ordered_max_write_buffer_number=3
-# The target number of write history bytes to hold in memory (default: 20MB)
-storage.rocksdb.max_write_buffer_size_to_maintain=20971520
-# rocksdb memtable prefix bloom size ratio (size=write_buffer_size*memtable_prefix_bloom_size_ratio)
-storage.rocksdb.memtable_prefix_bloom_size_ratio=0.1
-# dump rocksdb.stats to LOG every stats_dump_period_sec
-storage.rocksdb.stats_dump_period_sec=180
-# rocksdb perf level:
-#   0: kDisable
-#   1: kEnableCount
-#   2: kEnableTimeAndCPUTimeExceptForMutex
-#   3: kEnableTimeExceptForMutex
-#   4: kEnableTime
-# see also: https://github.com/facebook/rocksdb/wiki/Perf-Context-and-IO-Stats-Context#profile-levels-and-costs
-storage.rocksdb.perf_level=0
-# all rocksdb operations which latency greater than perf_slow_operation_us
-# will be considered a slow operation
-storage.rocksdb.perf_slow_us=100
-# rocksdb perf sampling ratio
-storage.rocksdb.perf_sampling_ratio=0
-# if the number of inode's s3chunkinfo exceed the limit_size,
-# we will sending its with rpc streaming instead of
-# padding its into inode (default: 25000, about 25000 * 41 (byte) = 1MB)
-storage.s3_meta_inside_inode.limit_size=25000
-
-# recycle options
-# metaserver scan recycle period, default 1h
-recycle.manager.scanPeriodSec=3600
-# metaserver recycle cleaner scan list dentry limit, default 1000
-recycle.cleaner.scanLimit=1000
-
-#### excutorOpt
-# excutorOpt rpc with metaserver
-# rpc retry times with metaserver
-excutorOpt.maxRetry=4294967295
-# internal rpc retry times with metaserver
-excutorOpt.maxInternalRetry = 3
-# Retry sleep time between failed RPCs
-excutorOpt.retryIntervalUS=100000
-# RPC timeout for communicating with metaserver
-excutorOpt.rpcTimeoutMS=1000
-# RPC stream idle timeout
-excutorOpt.rpcStreamIdleTimeoutMS=500
-# The maximum timeout RPC time of the retry request.
-# The timeout time will follow the exponential backoff policy.
-# Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
-excutorOpt.maxRPCTimeoutMS=8000
-# Maximum sleep time between retry requests.
-# when the network is congested or the metaserver is overloaded,
-# it is necessary to increase the sleep time
-excutorOpt.maxRetrySleepIntervalUS=8000000
-excutorOpt.minRetryTimesForceTimeoutBackoff=5
-excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
-# batch limit of get inode attr and xattr
-excutorOpt.batchInodeAttrLimit=10000
-excutorOpt.enableMultiMountPointRename=true
-
-#### metaCacheOpt
-# Gets the number of retries for the leader
-metaCacheOpt.metacacheGetLeaderRetry=3
-# Need to sleep for a period of time before each get leader retry
-metaCacheOpt.metacacheRPCRetryIntervalUS=100000
-# RPC timeout of get leader
-metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000

curvefs/conf/tools.conf

@@ -2,12 +2,8 @@
 mdsAddr=127.0.0.1:6700  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_port") }} __ANSIBLE_TEMPLATE__
 mdsDummyAddr=127.0.0.1:7700  # __CURVEADM_TEMPLATE__ ${cluster_mds_dummy_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_dummy_port") }} __ANSIBLE_TEMPLATE__
 # rpc timeout
-rpcTimeoutMs=30000
-rpcRetryTimes=3
-# rpc stream idle timeout
-rpcStreamIdleTimeoutMs=10000
-# rpc retry interval
-rpcRetryIntervalUs=1000
+rpcTimeoutMs=10000
+rpcRetryTimes=5
 # topo file path
 topoFilePath=curvefs/test/tools/topo_example.json  # __CURVEADM_TEMPLATE__ /curvefs/tools/conf/topology.json __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ project_root_dest }}/conf/topology.json __ANSIBLE_TEMPLATE__
 # metaserver external address
@@ -18,15 +14,11 @@ etcdAddr=127.0.0.1:12379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEAD
 blockSize=1048576
 fsType=s3
 # volume
-volumeSize=0
+volumeSize=1048576
 volumeBlockSize=4096
 volumeName=volume
 volumeUser=user
 volumePassword=password
-volumeBlockGroupSize=134217728
-volumeCluster=127.0.0.1:6666,127.0.0.1:6667,127.0.0.1:6668
-# support |AtStart| and |AtEnd|
-volumeBitmapLocation=AtStart
 # s3
 s3.ak=ak
 s3.sk=sk
@@ -34,13 +26,3 @@ s3.endpoint=endpoint
 s3.bucket_name=bucket
 s3.blocksize=4194304
 s3.chunksize=67108864
-s3.useVirtualAddressing=false
-# s3 objectPrefix, if set 0, means no prefix, if set 1, means inode prefix
-# if set 2 and other values mean hash prefix
-s3.objectPrefix=0
-# statistic info in xattr, hardlink will not be supported when enable
-enableSumInDir=true
-
-# fs recycle, if set 0, disable fs recycle, delete files directly,
-# if set not 0, enable fs recycle, delete files after a period of time
-recycleTimeHour=0

curvefs/docker/Dockerfile

@@ -0,0 +1,8 @@
+FROM opencurvedocker/curve-base:debian9
+ENV TZ=Asia/Shanghai
+RUN mkdir -p /curvefs /etc/curvefs /core
+COPY curvefs /curvefs
+COPY entrypoint.sh /
+COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+RUN chmod a+x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/base/Dockerfile

@@ -0,0 +1,28 @@
+FROM debian:9
+
+RUN echo "deb http://mirrors.163.com/debian/ stretch main\n" \
+         "deb http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ stretch/updates main non-free contrib\n" \
+         "deb http://httpredir.debian.org/debian stretch-backports main contrib non-free\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        gcc \
+        gdb \
+        make \
+        openssl \
+        net-tools \
+        libcurl3-gnutls \
+        podlators-perl \
+        linux-perf \
+        vim \
+        curl \
+        cron \
+        procps
+
+COPY fusermount3 /usr/local/bin
+COPY libetcdclient.so /usr/lib/
+COPY libfuse3.so libfuse3.so.3 libfuse3.so.3.10.5 /usr/lib/x86_64-linux-gnu/
+COPY libjemalloc.so libjemalloc.so.1 libjemalloc.so.2 /usr/local/lib/

curvefs/docker/debian9/Dockerfile

@@ -1,8 +0,0 @@
-FROM opencurvedocker/curve-base:debian9
-COPY entrypoint.sh /
-COPY curvefs /curvefs
-COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/
-RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
-    && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
-    && cp /curvefs/tools-v2/sbin/curve /usr/bin/
-ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/entrypoint.sh

@@ -8,7 +8,6 @@ g_args=""
 g_prefix=""
 g_binary=""
 g_start_args=""
-g_preexec="/curvefs/tools-v2/sbin/daemon"
 
 ############################  BASIC FUNCTIONS
 function msg() {
@@ -88,10 +87,6 @@ function prepare() {
             g_binary="$g_prefix/sbin/curve-fuse"
             g_start_args="--confPath $conf_path"
             ;;
-        monitor)
-            g_binary="python3"
-            g_start_args="target_json.py"
-            ;;
         *)
             usage
             exit 1
@@ -107,8 +102,6 @@ function create_directory() {
     chmod 700 "$g_prefix/data"
     if [ "$g_role" == "etcd" ]; then
         mkdir -p "$g_prefix/data/wal"
-    elif [ "$g_role" == "metaserver" ]; then
-        mkdir -p "$g_prefix/data/storage"
     elif [ "$g_role" == "client" ]; then
         mkdir -p "$g_prefix/mnt"
     fi
@@ -120,16 +113,7 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
-    [[ ! -z $g_preexec ]] && $g_preexec &
-    if [ $g_role == "etcd" ]; then
-        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
-    elif [ $g_role == "monitor" ]; then
-        cd $g_prefix
-        exec $g_binary $g_start_args
-    else
-        exec $g_binary $g_start_args
-    fi
-
+    exec $g_binary $g_start_args
 }
 
 ############################  MAIN()

curvefs/monitor/README.md

@@ -1,108 +0,0 @@
-# 目录结构介绍
-
-```
-monitor
-├── curve-monitor.sh        # curve集群监控的控制脚本，用于启动、停止、重启监控功能。
-├── docker-compose.yml      # 编排监控系统相关容器的配置文件，包括prometheus容器、grafana容器。
-|                           # 修改该文件来配置各组件的配置参数。
-├── grafana                 # grafana相关目录
-│   ├── dashboards          # grafana所有dashboards的json文件存放目录，grafana将从该目录加载文件来创建dashboards；
-|   |   |                   # 通过update_dashboard.sh脚本来更新最新的dashboards。
-│   │   ├── etcd.json
-│   │   ├── mds.json
-│   │   ├── metaserver.json
-│   │   └── clinet.json
-│   ├── grafana.ini         # grafana的启动配置文件，将映射到容器的 `/etc/grafana/grafana.ini` 上
-│   ├── provisioning        # grafana预配置相关目录，将映射到容器的`/etc/grafana/provisioning`上
-│   │   ├── dashboards
-│   │   │   └── all.yml
-│   │   └── datasources     # grafana的datasources的json文件存放目录，grafana将从该目录加载文件来创建datasources。
-│   │       └── all.yml
-│   └── report              # grafana日报临时目录，将映射到reporter容器的`/tmp/report`目录上
-│       └── README
-├── grafana-report.py
-├── prometheus              # prometheus相关目录
-│   ├── prometheus.yml      # prometheus的配置文件
-│   └── target.json
-├── README.md
-├── target.ini              # target_json.py脚本依赖的一些配置
-├── target_json.py          # 用于生成prometheus监控对象的python脚本，每隔一段时间用curvefs_tool拉取监控目标并更新。
-└── update_dashboard.sh     # 从grafana界面配置环境当中拉取最新的dashboard，用于更新该环境上grafana的界面。
-```
-
-## 使用说明
-
-### 环境初始化
-
-1.部署监控系统的机器需要安装如下组件：
-
-docker、docker-compose、jq
-
-* docker安装
-
-```
-$ curl -fsSL get.docker.com -o get-docker.sh
-$ sudo sh get-docker.sh --mirror Aliyun
-```
-
-或者直接安装
-
-```
-apt-get install docker-ce
-apt-get install docker-ce-cli
-```
-
-* docker-compose
-
-* ```
-  curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
-  chmod +x /usr/local/bin/docker-compose
-  ```
-
-  或者直接安装
-
-  ```
-  apt-get install docker-compose
-  ```
-
-* jq
-
-  update_dashboard.sh脚本需要依赖jq命令，这个一般机器上都没装
-
-  ```
-  apt-get install jq
-  ```
-
-### 部署监控系统
-
-* 修改相关配置
-
-1.修改target_json.py文件中相应的配置
-
-2.修改update_dashboard.sh，将 URL 和 LOGIN 改为对应的地址和用户名密码
-
-3.修改docker-compose.yml文件，主要是映射的目录路径
-
-* 启动docker-compose
-
-在当前目录下执行如下命令即可
-
-```curve-monitor.sh start ```
-
-# grafana每日报表
-
-每日报表需要设置定时任务，通过 grafana-report.py 来发送邮件。
-
-请修改 grafana-report.py 文件（13～25行）中的内容（包括发件人，收件人，用户名和密码等）。
-
-此外 grafana-report.py 的运行需要依赖一些第三方库，请参照文件内容安装相关库。
-
-```bash
-sudo apt install python-pip
-pip install email
-```
-
-crontab配置定时任务，添加如下任务：
-30 8 ** *python /etc/curve/monitor/grafana-report.py >> /etc/curve/monitor/cron.log 2>&1
-如果机器上没有配置其他的定时任务，可直接用下面命令
-echo "30 8 * * * python /etc/curve/monitor/grafana-report.py >> /etc/curve/monitor/cron.log 2>&1" >> conf && crontab conf && rm -f conf

curvefs/monitor/curve-monitor.sh

@@ -1,63 +0,0 @@
-#!/bin/sh
-
-#sh update_dashboard.sh
-#echo "update dashboards success!"
-
-WORKDIR=/etc/curvefs/monitor
-
-if [ ! -d $WORKDIR ]; then
-  echo "${WORKDIR} not exists"
-  exit 1
-fi
-
-cd $WORKDIR
-chmod -R 777 prometheus
-chmod -R 777 grafana
-
-start() {
-    echo "==========start==========="
-
-    echo "" > monitor.log
-
-    docker-compose up >> monitor.log 2>&1 &
-    echo "start metric system success!"
-}
-
-stop() {
-    echo "===========stop============"
-
-    docker-compose down
-
-    ID=`(ps -ef | grep "target_json.py"| grep -v "grep") | awk '{print $2}'`
-    for id in $ID
-    do
-        kill -9 $id
-	echo "killed $id"
-    done
-}
-
-restart() {
-    stop
-    echo "sleeping........."
-    sleep 3
-    start
-}
-
-case "$1" in
-    'start')
-        start
-        ;;
-    'stop')
-        stop
-        ;;
-    'status')
-        status
-        ;;
-    'restart')
-        restart
-        ;;
-    *)
-    echo "usage: $0 {start|stop|restart}"
-    exit 1
-        ;;
-    esac

curvefs/monitor/docker-compose.yml

@@ -1,40 +0,0 @@
-version: '2.0'
-
-services:
-
-  prometheus:
-    image: prom/prometheus:latest
-    volumes:
-      - ./prometheus/:/etc/prometheus/:rw
-      - ./prometheus/data:/prometheus:rw
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--storage.tsdb.retention.time=7d'
-      - '--storage.tsdb.retention.size=256GB'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
-      - '--web.listen-address=:9090'
-    network_mode: host
-
-  grafana:
-    image: grafana/grafana
-    depends_on:
-      - prometheus
-    network_mode: host
-    volumes:
-      - ./grafana/data:/var/lib/grafana:rw
-      - ./grafana/grafana.ini:/etc/grafana/grafana.ini:rw
-      - ./grafana/provisioning:/etc/grafana/provisioning:rw
-    environment:
-      - GF_INSTALL_PLUGINS=grafana-piechart-panel
-      - GF_SECURITY_ADMIN_USER=admin
-      - GF_SECURITY_ADMIN_PASSWORD=curve
-
-  reporter:
-    image: promoon/reporter:latest
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-      - /etc/timezone:/etc/timezone:ro
-      - ./grafana/report:/tmp/report:rw
-    network_mode: host

curvefs/monitor/grafana-report.py

@@ -1,117 +0,0 @@
-# coding: utf8
-
-import smtplib
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-from email.mime.image import MIMEImage
-from email.utils import parseaddr, formataddr
-import time as Time
-import re
-import shutil
-import os
-
-sender = 'Grafana<xxxxxxxxx@163.com>'
-to_address = ['xxxxxxxxx@163.com']
-username = 'xxxxxxxxx@163.com'
-password = 'xxxxxxxxx' # SMTP授权码
-smtpserver = 'xxxx.163.com:1234'
-sourcefile= '/etc/curvefs/monitor/grafana/report/report.tex'
-imagedir= '/etc/curvefs/monitor/grafana/report/images/'
-pdfpath= '/etc/curvefs/monitor/grafana/report/report.pdf'
-clustername = '【CURVE】xxxxxxxxx'
-grafanauri = '127.0.0.1:3000'
-reporteruri = '127.0.0.1:8686'
-dashboardid = 'xxxxxxxxx'
-apitoken = 'xxxxxxxxx'
-
-def get_images():
-    image_name_list = []
-    file = open(sourcefile, 'r')
-    line = file.readline()
-    while line:
-        # print (line)
-        prefix_image_name = re.findall(r'image\d+', line)
-        if prefix_image_name:
-            print (prefix_image_name)
-            image_name_list.append(prefix_image_name[0])
-        line = file.readline()
-    file.close()
-
-    return image_name_list
-
-def getMsgImage(image_name):
-    file_name = imagedir+image_name+'.png'
-    print (file_name)
-    fp = open(file_name, 'rb')
-    msgImage = MIMEImage(fp.read())
-    fp.close()
-    msgImage.add_header('Content-ID', image_name)
-    msgImage.add_header("Content-Disposition", "inline", filename=file_name)
-    return msgImage
-
-def attach_body(msgRoot):
-    image_list = get_images()
-
-    image_body = ""
-    for image in image_list:
-        image_body += ('<img src="cid:%s" alt="%s">' % (image, image))
-        msgRoot.attach(getMsgImage(image))
-
-    html_str = '<html><head><style>#string{text-align:center;font-size:25px;}</style></head><body>%s</body></html>' % (image_body)
-
-    mailMsg = """
-    <p>可点击如下链接在grafana面板中查看（若显示混乱，请在附件pdf中查看）</p>
-    <p><a href="http://%s">grafana链接</a></p>
-    """ % (grafanauri)
-    mailMsg += html_str
-    print(mailMsg)
-    content = MIMEText(mailMsg,'html','utf-8')
-    msgRoot.attach(content)
-
-# 发送dashboard日报邮件
-def send_mail():
-    time_now = int(Time.time())
-    time_local = Time.localtime(time_now)
-    dt = Time.strftime("%Y%m%d",time_local)
-
-    msgRoot = MIMEMultipart('related')
-    msgRoot['Subject'] = '%s集群监控日报-%s' % (clustername, dt)
-    msgRoot['From'] = sender
-    msgRoot['To'] = ",".join( to_address ) # 发给多人
-
-    # 添加pdf附件
-    pdf_attach = MIMEText(open(pdfpath, 'rb').read(), 'base64', 'utf-8')
-    pdf_attach["Content-Type"] = 'application/octet-stream'
-    # 这里的filename可以任意写，写什么名字，邮件中显示什么名字
-    pdf_attach["Content-Disposition"] = 'attachment; filename="reporter-{}.pdf"'.format(dt)
-    msgRoot.attach(pdf_attach)
-
-    # 添加正文
-    attach_body(msgRoot)
-
-    smtp = smtplib.SMTP_SSL(smtpserver)
-    smtp.login(username, password)
-    smtp.sendmail(sender, to_address, msgRoot.as_string())
-    smtp.quit()
-
-def clear():
-    shutil.rmtree(imagedir)
-    os.mkdir(imagedir)
-    os.chmod(imagedir, 0777)
-
-def generate_report():
-    downloadcmd = (
-        "wget -O %s "
-        "http://%s/api/v5/report/%s?apitoken=%s"
-        "\&from=now-24h\&to=now"
-    ) % (pdfpath, reporteruri, dashboardid, apitoken)
-    print(downloadcmd)
-    os.system(downloadcmd)
-
-def main():
-    generate_report()
-    send_mail()
-    clear()
-
-if __name__ == '__main__':
-    main()

curvefs/monitor/grafana/grafana.ini

@@ -1,579 +0,0 @@
-##################### Grafana Configuration Example #####################
-#
-# Everything has defaults so you only need to uncomment things you want to
-# change
-
-# possible values : production, development
-;app_mode = production
-
-# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
-;instance_name = ${HOSTNAME}
-
-#################################### Paths ####################################
-[paths]
-# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
-;data = /var/lib/grafana
-
-# Temporary files in `data` directory older than given duration will be removed
-;temp_data_lifetime = 24h
-
-# Directory where grafana can store logs
-;logs = /var/log/grafana
-
-# Directory where grafana will automatically scan and look for plugins
-;plugins = /var/lib/grafana/plugins
-
-# folder that contains provisioning config files that grafana will apply on startup and while running.
-;provisioning = conf/provisioning
-
-#################################### Server ####################################
-[server]
-# Protocol (http, https, socket)
-;protocol = http
-
-# The ip address to bind to, empty will bind to all interfaces
-;http_addr =
-
-# The http port  to use
-;http_port = 3000
-
-# The public facing domain name used to access grafana from a browser
-;domain = localhost
-
-# Redirect to correct domain if host header does not match domain
-# Prevents DNS rebinding attacks
-;enforce_domain = false
-
-# The full public facing url you use in browser, used for redirects and emails
-# If you use reverse proxy and sub path specify full url (with sub path)
-;root_url = http://localhost:3000
-
-# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
-;serve_from_sub_path = false
-
-# Log web requests
-;router_logging = false
-
-# the path relative working path
-;static_root_path = public
-
-# enable gzip
-;enable_gzip = false
-
-# https certs & key file
-;cert_file =
-;cert_key =
-
-# Unix socket path
-;socket =
-
-#################################### Database ####################################
-[database]
-# You can configure the database connection by specifying type, host, name, user and password
-# as separate properties or as on string using the url properties.
-
-# Either "mysql", "postgres" or "sqlite3", it's your choice
-;type = sqlite3
-;host = 127.0.0.1:3306
-;name = grafana
-;user = root
-# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
-;password =
-
-# Use either URL or the previous fields to configure the database
-# Example: mysql://user:secret@host:port/database
-;url =
-
-# For "postgres" only, either "disable", "require" or "verify-full"
-;ssl_mode = disable
-
-# For "sqlite3" only, path relative to data_path setting
-;path = grafana.db
-
-# Max idle conn setting default is 2
-;max_idle_conn = 2
-
-# Max conn setting default is 0 (mean not set)
-;max_open_conn =
-
-# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
-;conn_max_lifetime = 14400
-
-# Set to true to log the sql calls and execution times.
-;log_queries =
-
-# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared)
-;cache_mode = private
-
-#################################### Cache server #############################
-[remote_cache]
-# Either "redis", "memcached" or "database" default is "database"
-;type = database
-
-# cache connectionstring options
-# database: will use Grafana primary database.
-# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0`. Only addr is required.
-# memcache: 127.0.0.1:11211
-;connstr =
-
-#################################### Data proxy ###########################
-[dataproxy]
-
-# This enables data proxy logging, default is false
-;logging = false
-
-# How long the data proxy should wait before timing out default is 30 (seconds)
-;timeout = 30
-
-# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
-;send_user_header = false
-
-#################################### Analytics ####################################
-[analytics]
-# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
-# No ip addresses are being tracked, only simple counters to track
-# running instances, dashboard and error counts. It is very helpful to us.
-# Change this option to false to disable reporting.
-;reporting_enabled = true
-
-# Set to false to disable all checks to https://grafana.net
-# for new vesions (grafana itself and plugins), check is used
-# in some UI views to notify that grafana or plugin update exists
-# This option does not cause any auto updates, nor send any information
-# only a GET request to http://grafana.com to get latest versions
-;check_for_updates = true
-
-# Google Analytics universal tracking code, only enabled if you specify an id here
-;google_analytics_ua_id =
-
-# Google Tag Manager ID, only enabled if you specify an id here
-;google_tag_manager_id =
-
-#################################### Security ####################################
-[security]
-# default admin user, created on startup
-;admin_user = admin
-
-# default admin password, can be changed before first start of grafana,  or in profile settings
-;admin_password = admin
-
-# used for signing
-;secret_key = SW2YcwTIb9zpOOhoPsMm
-
-# disable gravatar profile images
-;disable_gravatar = false
-
-# data source proxy whitelist (ip_or_domain:port separated by spaces)
-;data_source_proxy_whitelist =
-
-# disable protection against brute force login attempts
-;disable_brute_force_login_protection = false
-
-# set to true if you host Grafana behind HTTPS. default is false.
-;cookie_secure = false
-
-# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none"
-;cookie_samesite = lax
-
-# set to true if you want to allow browsers to render Grafana in a <frame>, <iframe>, <embed> or <object>. default is false.
-;allow_embedding = false
-
-# Set to true if you want to enable http strict transport security (HSTS) response header.
-# This is only sent when HTTPS is enabled in this configuration.
-# HSTS tells browsers that the site should only be accessed using HTTPS.
-# The default version will change to true in the next minor release, 6.3.
-;strict_transport_security = false
-
-# Sets how long a browser should cache HSTS. Only applied if strict_transport_security is enabled.
-;strict_transport_security_max_age_seconds = 86400
-
-# Set to true if to enable HSTS preloading option. Only applied if strict_transport_security is enabled.
-;strict_transport_security_preload = false
-
-# Set to true if to enable the HSTS includeSubDomains option. Only applied if strict_transport_security is enabled.
-;strict_transport_security_subdomains = false
-
-# Set to true to enable the X-Content-Type-Options response header.
-# The X-Content-Type-Options response HTTP header is a marker used by the server to indicate that the MIME types advertised
-# in the Content-Type headers should not be changed and be followed. The default will change to true in the next minor release, 6.3.
-;x_content_type_options = false
-
-# Set to true to enable the X-XSS-Protection header, which tells browsers to stop pages from loading
-# when they detect reflected cross-site scripting (XSS) attacks. The default will change to true in the next minor release, 6.3.
-;x_xss_protection = false
-
-#################################### Snapshots ###########################
-[snapshots]
-# snapshot sharing options
-;external_enabled = true
-;external_snapshot_url = https://snapshots-origin.raintank.io
-;external_snapshot_name = Publish to snapshot.raintank.io
-
-# remove expired snapshot
-;snapshot_remove_expired = true
-
-#################################### Dashboards History ##################
-[dashboards]
-# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
-;versions_to_keep = 20
-
-#################################### Users ###############################
-[users]
-# disable user signup / registration
-;allow_sign_up = true
-
-# Allow non admin users to create organizations
-;allow_org_create = true
-
-# Set to true to automatically assign new users to the default organization (id 1)
-;auto_assign_org = true
-
-# Default role new users will be automatically assigned (if disabled above is set to true)
-;auto_assign_org_role = Viewer
-
-# Background text for the user field on the login page
-;login_hint = email or username
-;password_hint = password
-
-# Default UI theme ("dark" or "light")
-;default_theme = dark
-
-# External user management, these options affect the organization users view
-;external_manage_link_url =
-;external_manage_link_name =
-;external_manage_info =
-
-# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
-;viewers_can_edit = false
-
-# Editors can administrate dashboard, folders and teams they create
-;editors_can_admin = false
-
-[auth]
-# Login cookie name
-;login_cookie_name = grafana_session
-
-# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days,
-;login_maximum_inactive_lifetime_days = 7
-
-# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days.
-;login_maximum_lifetime_days = 30
-
-# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes.
-;token_rotation_interval_minutes = 10
-
-# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
-;disable_login_form = false
-
-# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
-;disable_signout_menu = false
-
-# URL to redirect the user to after sign out
-;signout_redirect_url =
-
-# Set to true to attempt login with OAuth automatically, skipping the login screen.
-# This setting is ignored if multiple OAuth providers are configured.
-;oauth_auto_login = false
-
-#################################### Anonymous Auth ######################
-[auth.anonymous]
-# enable anonymous access
-;enabled = false
-
-# specify organization name that should be used for unauthenticated users
-;org_name = Main Org.
-
-# specify role for unauthenticated users
-;org_role = Viewer
-
-#################################### Github Auth ##########################
-[auth.github]
-;enabled = false
-;allow_sign_up = true
-;client_id = some_id
-;client_secret = some_secret
-;scopes = user:email,read:org
-;auth_url = https://github.com/login/oauth/authorize
-;token_url = https://github.com/login/oauth/access_token
-;api_url = https://api.github.com/user
-;team_ids =
-;allowed_organizations =
-
-#################################### Google Auth ##########################
-[auth.google]
-;enabled = false
-;allow_sign_up = true
-;client_id = some_client_id
-;client_secret = some_client_secret
-;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
-;auth_url = https://accounts.google.com/o/oauth2/auth
-;token_url = https://accounts.google.com/o/oauth2/token
-;api_url = https://www.googleapis.com/oauth2/v1/userinfo
-;allowed_domains =
-
-#################################### Generic OAuth ##########################
-[auth.generic_oauth]
-;enabled = false
-;name = OAuth
-;allow_sign_up = true
-;client_id = some_id
-;client_secret = some_secret
-;scopes = user:email,read:org
-;auth_url = https://foo.bar/login/oauth/authorize
-;token_url = https://foo.bar/login/oauth/access_token
-;api_url = https://foo.bar/user
-;team_ids =
-;allowed_organizations =
-;tls_skip_verify_insecure = false
-;tls_client_cert =
-;tls_client_key =
-;tls_client_ca =
-
-; Set to true to enable sending client_id and client_secret via POST body instead of Basic authentication HTTP header
-; This might be required if the OAuth provider is not RFC6749 compliant, only supporting credentials passed via POST payload
-;send_client_credentials_via_post = false
-
-#################################### SAML Auth ###########################
-;[auth.saml] # Enterprise only
-;enabled = false
-;private_key =
-;private_key_path =
-;certificate =
-;certificate_path =
-;idp_metadata =
-;idp_metadata_path =
-;idp_metadata_url =
-;max_issue_delay = 90s
-;metadata_valid_duration = 48h
-
-#################################### Grafana.com Auth ####################
-[auth.grafana_com]
-;enabled = false
-;allow_sign_up = true
-;client_id = some_id
-;client_secret = some_secret
-;scopes = user:email
-;allowed_organizations =
-
-#################################### Auth Proxy ##########################
-[auth.proxy]
-;enabled = false
-;header_name = X-WEBAUTH-USER
-;header_property = username
-;auto_sign_up = true
-;ldap_sync_ttl = 60
-;whitelist = 192.168.1.1, 192.168.2.1
-;headers = Email:X-User-Email, Name:X-User-Name
-
-#################################### Basic Auth ##########################
-[auth.basic]
-;enabled = true
-
-#################################### Auth LDAP ##########################
-[auth.ldap]
-;enabled = false
-;config_file = /etc/grafana/ldap.toml
-;allow_sign_up = true
-
-# LDAP backround sync (Enterprise only)
-# At 1 am every day
-;sync_cron = "0 0 1 * * *"
-;active_sync_enabled = true
-
-#################################### SMTP / Emailing ##########################
-[smtp]
-;enabled = false
-;host = localhost:25
-;user =
-# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
-;password =
-;cert_file =
-;key_file =
-;skip_verify = false
-;from_address = admin@grafana.localhost
-;from_name = Grafana
-# EHLO identity in SMTP dialog (defaults to instance_name)
-;ehlo_identity = dashboard.example.com
-
-[emails]
-;welcome_email_on_sign_up = false
-
-#################################### Logging ##########################
-[log]
-# Either "console", "file", "syslog". Default is console and  file
-# Use space to separate multiple modes, e.g. "console file"
-;mode = console file
-
-# Either "debug", "info", "warn", "error", "critical", default is "info"
-;level = info
-
-# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
-;filters =
-
-# For "console" mode only
-[log.console]
-;level =
-
-# log line format, valid options are text, console and json
-;format = console
-
-# For "file" mode only
-[log.file]
-;level =
-
-# log line format, valid options are text, console and json
-;format = text
-
-# This enables automated log rotate(switch of following options), default is true
-;log_rotate = true
-
-# Max line number of single file, default is 1000000
-;max_lines = 1000000
-
-# Max size shift of single file, default is 28 means 1 << 28, 256MB
-;max_size_shift = 28
-
-# Segment log daily, default is true
-;daily_rotate = true
-
-# Expired days of log file(delete after max days), default is 7
-;max_days = 7
-
-[log.syslog]
-;level =
-
-# log line format, valid options are text, console and json
-;format = text
-
-# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
-;network =
-;address =
-
-# Syslog facility. user, daemon and local0 through local7 are valid.
-;facility =
-
-# Syslog tag. By default, the process' argv[0] is used.
-;tag =
-
-#################################### Alerting ############################
-[alerting]
-# Disable alerting engine & UI features
-;enabled = true
-# Makes it possible to turn off alert rule execution but alerting UI is visible
-;execute_alerts = true
-
-# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
-;error_or_timeout = alerting
-
-# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
-;nodata_or_nullvalues = no_data
-
-# Alert notifications can include images, but rendering many images at the same time can overload the server
-# This limit will protect the server from render overloading and make sure notifications are sent out quickly
-;concurrent_render_limit = 5
-
-
-# Default setting for alert calculation timeout. Default value is 30
-;evaluation_timeout_seconds = 30
-
-# Default setting for alert notification timeout. Default value is 30
-;notification_timeout_seconds = 30
-
-# Default setting for max attempts to sending alert notifications. Default value is 3
-;max_attempts = 3
-
-#################################### Explore #############################
-[explore]
-# Enable the Explore section
-;enabled = true
-
-#################################### Internal Grafana Metrics ##########################
-# Metrics available at HTTP API Url /metrics
-[metrics]
-# Disable / Enable internal metrics
-;enabled           = true
-
-# Publish interval
-;interval_seconds  = 10
-
-# Send internal metrics to Graphite
-[metrics.graphite]
-# Enable by setting the address setting (ex localhost:2003)
-;address =
-;prefix = prod.grafana.%(instance_name)s.
-
-#################################### Distributed tracing ############
-[tracing.jaeger]
-# Enable by setting the address sending traces to jaeger (ex localhost:6831)
-;address = localhost:6831
-# Tag that will always be included in when creating new spans. ex (tag1:value1,tag2:value2)
-;always_included_tag = tag1:value1
-# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote
-;sampler_type = const
-# jaeger samplerconfig param
-# for "const" sampler, 0 or 1 for always false/true respectively
-# for "probabilistic" sampler, a probability between 0 and 1
-# for "rateLimiting" sampler, the number of spans per second
-# for "remote" sampler, param is the same as for "probabilistic"
-# and indicates the initial sampling rate before the actual one
-# is received from the mothership
-;sampler_param = 1
-# Whether or not to use Zipkin propagation (x-b3- HTTP headers).
-;zipkin_propagation = false
-# Setting this to true disables shared RPC spans.
-# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
-;disable_shared_zipkin_spans = false
-
-#################################### Grafana.com integration  ##########################
-# Url used to import dashboards directly from Grafana.com
-[grafana_com]
-;url = https://grafana.com
-
-#################################### External image storage ##########################
-[external_image_storage]
-# Used for uploading images to public servers so they can be included in slack/email messages.
-# you can choose between (s3, webdav, gcs, azure_blob, local)
-;provider =
-
-[external_image_storage.s3]
-;bucket =
-;region =
-;path =
-;access_key =
-;secret_key =
-
-[external_image_storage.webdav]
-;url =
-;public_url =
-;username =
-;password =
-
-[external_image_storage.gcs]
-;key_file =
-;bucket =
-;path =
-
-[external_image_storage.azure_blob]
-;account_name =
-;account_key =
-;container_name =
-
-[external_image_storage.local]
-# does not require any configuration
-
-[rendering]
-# Options to configure external image rendering server like https://github.com/grafana/grafana-image-renderer
-;server_url =
-;callback_url =
-
-[enterprise]
-# Path to a valid Grafana Enterprise license.jwt file
-;license_path =
-
-[panels]
-# If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.
-;disable_sanitize_html = false
-
-[plugins]
-;enable_alpha = false
-;app_tls_skip_verify_insecure = false

curvefs/monitor/grafana/provisioning/dashboards/all.yml

@@ -1,6 +0,0 @@
-- name: 'default'
-  org_id: 1
-  folder: ''
-  type: 'file'
-  options:
-    folder: '/etc/grafana/provisioning/dashboards'