curvefs/fix: addition of the objectPrefix field in the message S3Info has caused compatibility issues

Signed-off-by: ilixiaocui <ilixiaocui@163.com>

curvefs/conf/client.conf

@@ -135,14 +135,14 @@ fuseClient.throttle.burstReadIopsSecs=180
 #### filesystem metadata
 # {
 # fs.disableXattr:
-#   if you want to get curvefs specified xattr,
-#   you can mount another fs with |fs.disableXattr| is true
+#   if you want to get better metadata performance,
+#   you can mount fs with |fs.disableXattr| is true
 #
 # fs.lookupCache.negativeTimeoutSec:
 #   entry which not found will be cached if |timeout| > 0
 fs.cto=true
 fs.maxNameLength=255
-fs.disableXattr=true
+fs.disableXattr=false
 fs.accessLogging=true
 fs.kernelCache.attrTimeoutSec=3600
 fs.kernelCache.dirAttrTimeoutSec=3600

curvefs/src/client/filesystem/rpc_client.cpp

@@ -104,9 +104,11 @@ CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
             continue;
         }
 
+        // NOTE: we can't use std::move() for attribute for hard link
+        // which will sharing inode attribute.
         dirEntry.ino = ino;
         dirEntry.name = std::move(dentry.name());
-        dirEntry.attr = std::move(iter->second);
+        dirEntry.attr = iter->second;
         (*entries)->Add(dirEntry);
     }
     return CURVEFS_ERROR::OK;

curvefs/src/client/fuse_client.cpp

@@ -219,6 +219,7 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                      struct fuse_conn_info *conn) {
     (void)userdata;
     (void)conn;
+    fs_->Run();
     return CURVEFS_ERROR::OK;
 }
 
@@ -368,10 +369,7 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
         if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
             inodeManager_->ShipToFlush(parentInodeWrapper);
         } else {
-            ret = parentInodeWrapper->SyncAttr();
-            if (ret != CURVEFS_ERROR::OK) {
-                return CURVEFS_ERROR::OK;
-            }
+            return parentInodeWrapper->SyncAttr();
         }
     }
 

curvefs/src/client/warmup/warmup_manager.cpp

@@ -473,7 +473,7 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
             }
             if (context->retCode == 0) {
                 VLOG(9) << "Get Object success: " << context->key;
-                PutObjectToCache(key, context->key, context->buf, context->len);
+                PutObjectToCache(key, context);
                 CollectMetrics(&warmupS3Metric_.warmupS3Cached, context->len,
                                start);
                 warmupS3Metric_.warmupS3CacheSize << context->len;
@@ -481,7 +481,6 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
                     VLOG(6) << "pendingReq is over";
                     cond.Signal();
                 }
-                delete[] context->buf;
                 return;
             }
             warmupS3Metric_.warmupS3Cached.eps.count << 1;
@@ -678,9 +677,8 @@ void WarmupManagerS3Impl::AddFetchS3objectsTask(fuse_ino_t key,
     }
 }
 
-void WarmupManagerS3Impl::PutObjectToCache(fuse_ino_t key,
-                                           const std::string &filename,
-                                           const char *data, uint64_t len) {
+void WarmupManagerS3Impl::PutObjectToCache(
+    fuse_ino_t key, const std::shared_ptr<GetObjectAsyncContext> &context) {
     ReadLockGuard lock(inode2ProgressMutex_);
     auto iter = FindWarmupProgressByKeyLocked(key);
     if (iter == inode2Progress_.end()) {
@@ -692,17 +690,21 @@ void WarmupManagerS3Impl::PutObjectToCache(fuse_ino_t key,
     iter->second.FinishedPlusOne();
     switch (iter->second.GetStorageType()) {
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeDisk:
-        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(filename, data,
-                                                                 len);
+        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
+            context->key, context->buf, context->len);
         if (ret < 0) {
             LOG_EVERY_SECOND(INFO)
-                << "write read directly failed, key: " << filename;
+                << "write read directly failed, key: " << context->key;
         }
+        delete[] context->buf;
         break;
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeKvClient:
         if (kvClientManager_ != nullptr) {
-            kvClientManager_->Set(
-                std::make_shared<SetKVCacheTask>(filename, data, len));
+            kvClientManager_->Set(std::make_shared<SetKVCacheTask>(
+                context->key, context->buf, context->len,
+                [context](const std::shared_ptr<SetKVCacheTask> &) {
+                    delete[] context->buf;
+                }));
         }
         break;
     default:

curvefs/src/client/warmup/warmup_manager.h

@@ -413,8 +413,9 @@ class WarmupManagerS3Impl : public WarmupManager {
 
     void AddFetchS3objectsTask(fuse_ino_t key, std::function<void()> task);
 
-    void PutObjectToCache(fuse_ino_t key, const std::string &filename,
-                          const char *data, uint64_t len);
+    void
+    PutObjectToCache(fuse_ino_t key,
+                     const std::shared_ptr<GetObjectAsyncContext> &context);
 
  protected:
     std::deque<WarmupFilelist> warmupFilelistDeque_;

curvefs/src/mds/fs_storage.cpp

@@ -359,6 +359,15 @@ bool PersisKVStorage::LoadAllFs() {
 
         idToName_.emplace(fsInfo.fsid(), fsInfo.fsname());
 
+        // For compatibility when upgrading, the new field 'optional
+        // objectPrefix` in message `S3Info` needs to be set to the default
+        // value
+        if (fsInfo.fstype() == FSType::TYPE_S3) {
+            if (!fsInfo.detail().s3info().has_objectprefix()) {
+                fsInfo.mutable_detail()->mutable_s3info()->set_objectprefix(0);
+            }
+        }
+
         fs_.emplace(fsInfo.fsname(), std::move(fsInfo));
     }
 

curvefs/src/metaserver/trash.cpp

@@ -208,6 +208,21 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
         clientAdaptorOption.objectPrefix = s3Info.objectprefix();
         s3Adaptor_->Reinit(clientAdaptorOption, s3Info.ak(), s3Info.sk(),
             s3Info.endpoint(), s3Info.bucketname());
+        ret = inodeStorage_->PaddingInodeS3ChunkInfo(item.fsId,
+          item.inodeId, inode.mutable_s3chunkinfomap());
+        if (ret != MetaStatusCode::OK) {
+            LOG(ERROR) << "GetInode chunklist fail, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId
+                << ", retCode = " << MetaStatusCode_Name(ret);
+            return ret;
+        }
+        if (inode.s3chunkinfomap().empty()) {
+            LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId;
+            return MetaStatusCode::NOT_FOUND;
+        }
+        VLOG(9) << "DeleteInodeAndData, inode: "
+            << inode.ShortDebugString();
         int retVal = s3Adaptor_->Delete(inode);
         if (retVal != 0) {
             LOG(ERROR) << "S3ClientAdaptor delete s3 data failed"
@@ -216,7 +231,6 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
             return MetaStatusCode::S3_DELETE_ERR;
         }
     }
-
     ret = inodeStorage_->Delete(Key4Inode(item.fsId, item.inodeId));
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "Delete Inode fail, fsId = " << item.fsId

curvefs/test/client/test_fuse_volume_client.cpp

@@ -211,6 +211,8 @@ TEST_F(TestFuseVolumeClient, FuseOpInit_when_fs_exist) {
 
     ASSERT_EQ(fsInfo->fsid(), fsInfoExp.fsid());
     ASSERT_EQ(fsInfo->fsname(), fsInfoExp.fsname());
+
+    client_->GetFileSystem()->Destory();
 }
 
 TEST_F(TestFuseVolumeClient, FuseOpDestroy) {

curvefs/test/mds/persist_kvstorage_test.cpp

@@ -170,6 +170,10 @@ TEST_F(PersistKVStorageTest, TestInit) {
 
         EXPECT_FALSE(storage.Exist(3));
         EXPECT_FALSE(storage.Exist("foo"));
+
+        FsInfoWrapper wrapper;
+        EXPECT_EQ(FSStatusCode::OK, storage.Get("world", &wrapper));
+        EXPECT_TRUE(wrapper.GetFsDetail().s3info().has_objectprefix());
     }
 }
 
@@ -531,6 +535,5 @@ TEST_F(PersistKVStorageTest, TestDelete) {
         EXPECT_TRUE(storage.Exist(2));
     }
 }
-
 }  // namespace mds
 }  // namespace curvefs

curvefs/test/metaserver/BUILD

@@ -26,7 +26,6 @@ cc_test(
             "mock_metaserver_s3.h",
             "metaserver_s3_adaptor_test.h",
             "metaserver_s3_adaptor_test.cpp",
-            "mock_metaserver_s3_adaptor.h",
             "metaserver_s3_test.cpp",
             "mock_s3compact_inode.h",
             "s3compact_test.cpp",

curvefs/test/metaserver/trash_test.cpp

@@ -29,6 +29,7 @@
 #include "curvefs/test/metaserver/storage/utils.h"
 #include "src/fs/ext4_filesystem_impl.h"
 #include "curvefs/test/client/rpcclient/mock_mds_client.h"
+#include "curvefs/test/metaserver/mock_metaserver_s3_adaptor.h"
 
 using ::testing::AtLeast;
 using ::testing::StrEq;
@@ -105,7 +106,29 @@ class TestTrash : public ::testing::Test {
         inode.set_gid(0);
         inode.set_mode(0);
         inode.set_nlink(0);
-        inode.set_type(FsFileType::TYPE_FILE);
+        inode.set_type(FsFileType::TYPE_S3);
+        return inode;
+    }
+
+    Inode GenInodeHasChunks(uint32_t fsId, uint64_t inodeId) {
+        Inode inode;
+        inode.set_fsid(fsId);
+        inode.set_inodeid(inodeId);
+        inode.set_length(4096);
+        inode.set_ctime(0);
+        inode.set_ctime_ns(0);
+        inode.set_mtime(0);
+        inode.set_mtime_ns(0);
+        inode.set_atime(0);
+        inode.set_atime_ns(0);
+        inode.set_uid(0);
+        inode.set_gid(0);
+        inode.set_mode(0);
+        inode.set_nlink(0);
+        inode.set_type(FsFileType::TYPE_S3);
+
+        S3ChunkInfoList s3ChunkInfoList;
+        inode.mutable_s3chunkinfomap()->insert({0, s3ChunkInfoList});
         return inode;
     }
 
@@ -121,18 +144,17 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
     option.scanPeriodSec = 1;
     option.expiredAfterSec = 1;
     option.mdsClient = std::make_shared<MockMdsClient>();
-
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
     trashManager_->Init(option);
     trashManager_->Run();
-
     auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
     auto trash2 = std::make_shared<TrashImpl>(inodeStorage_);
     trashManager_->Add(1, trash1);
     trashManager_->Add(2, trash2);
 
-    inodeStorage_->Insert(GenInode(1, 1));
-    inodeStorage_->Insert(GenInode(1, 2));
-    inodeStorage_->Insert(GenInode(2, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 2));
+    inodeStorage_->Insert(GenInodeHasChunks(2, 1));
 
     ASSERT_EQ(inodeStorage_->Size(), 3);
 
@@ -141,17 +163,42 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
     trash2->Add(2, 1, 0);
 
     std::this_thread::sleep_for(std::chrono::seconds(5));
-
     std::list<TrashItem> list;
 
     trashManager_->ListItems(&list);
 
     ASSERT_EQ(0, list.size());
-
     ASSERT_EQ(inodeStorage_->Size(), 0);
 
     trashManager_->Fini();
 }
 
+TEST_F(TestTrash, testAdd3ItemAndNoDelete) {
+    TrashOption option;
+    option.scanPeriodSec = 1;
+    option.expiredAfterSec = 1;
+    option.mdsClient = std::make_shared<MockMdsClient>();
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
+    trashManager_->Init(option);
+    trashManager_->Run();
+
+    auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
+    trashManager_->Add(1, trash1);
+
+    inodeStorage_->Insert(GenInode(1, 1));
+    inodeStorage_->Insert(GenInode(1, 2));
+    inodeStorage_->Insert(GenInode(2, 1));
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trash1->Add(1, 1, 0);
+    trash1->Add(1, 2, 0);
+    std::this_thread::sleep_for(std::chrono::seconds(5));
+    std::list<TrashItem> list;
+
+    trashManager_->ListItems(&list);
+    ASSERT_EQ(0, list.size());
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trashManager_->Fini();
+}
+
 }  // namespace metaserver
 }  // namespace curvefs