diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 6de45461fb..7de6846835 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -50,6 +50,7 @@ #include "fdbclient/S3BlobStore.h" #include "fdbclient/SystemData.h" #include "fdbclient/json_spirit/json_spirit_writer_template.h" +#include "fdbclient/BackupContainer.h" #include "flow/Platform.h" @@ -1612,6 +1613,32 @@ ACTOR Future getLayerStatus(Reference tr wait(waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); + state std::vector> encryptionSetupResults; + state std::vector encryptionContainerIndices; + + for (int i = 0; i < tagContainers.size(); i++) { + if (tagContainers[i].get()->getEncryptionKeyFileName().present()) { + encryptionSetupResults.push_back(tagContainers[i].get()->encryptionSetupComplete()); + encryptionContainerIndices.push_back(i); + } + } + wait(waitForAllReady(encryptionSetupResults)); + json_spirit::mArray keysArr; + std::unordered_set seenKeyPaths; + for (int j = 0; j < encryptionContainerIndices.size() && j < 1e6; j++) { + int i = encryptionContainerIndices[j]; + std::string keyPath = tagContainers[i].get()->getEncryptionKeyFileName().get(); + + if (seenKeyPaths.find(keyPath) == seenKeyPaths.end()) { + seenKeyPaths.insert(keyPath); + json_spirit::mObject keyObj; + keyObj["path"] = tagContainers[i].get()->getEncryptionKeyFileName().get(); + keyObj["success"] = !encryptionSetupResults[j].isError(); + keysArr.push_back(keyObj); + } + } + o.create("encryption_keys") = keysArr; + JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest"); layerRoot.create("tags.timestamp") = now(); layerRoot.create("total_workers.$sum") = @@ -1640,7 +1667,11 @@ ACTOR Future getLayerStatus(Reference tr tagRoot.create("range_bytes_written") = tagRangeBytes[j].get(); tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get(); tagRoot.create("mutation_stream_id") = backupTagUids[j].toString(); - + tagRoot.create("file_level_encryption") = + tagContainers[j].get()->getEncryptionKeyFileName().present() ? true : false; + if (tagContainers[j].get()->getEncryptionKeyFileName().present()) { + tagRoot.create("encryption_key_file") = tagContainers[j].get()->getEncryptionKeyFileName().get(); + } j++; } } else if (exe == ProgramExe::DR_AGENT) { diff --git a/fdbbackup/tests/dir_backup_test.sh b/fdbbackup/tests/dir_backup_test.sh index 41af23c553..e752586fe0 100755 --- a/fdbbackup/tests/dir_backup_test.sh +++ b/fdbbackup/tests/dir_backup_test.sh @@ -104,6 +104,9 @@ function test_dir_backup_and_restore { err "Failed backup" return 1 fi + + test_fdbcli_status_json_for_bkup "${local_build_dir}" "${scratch_dir}" + log "Clear fdb data" if ! clear_data "${local_build_dir}" "${scratch_dir}"; then err "Failed clear data in fdb" diff --git a/fdbbackup/tests/s3_backup_test.sh b/fdbbackup/tests/s3_backup_test.sh index 15bf6b9c9c..2fd768cd3b 100755 --- a/fdbbackup/tests/s3_backup_test.sh +++ b/fdbbackup/tests/s3_backup_test.sh @@ -132,6 +132,9 @@ function test_s3_backup_and_restore { err "Failed backup" return 1 fi + + test_fdbcli_status_json_for_bkup "${local_build_dir}" "${local_scratch_dir}" + log "Clear fdb data" if ! clear_data "${local_build_dir}" "${local_scratch_dir}"; then err "Failed clear data in fdb" diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 0eb1560406..0ec137223f 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -1307,16 +1307,15 @@ public: 0400)); keyFile = _keyFile; } catch (Error& e) { - TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile") - .error(e) - .detail("FileName", encryptionKeyFileName); + TraceEvent(SevError, "FailedToOpenEncryptionKeyFile").error(e).detail("FileName", encryptionKeyFileName); throw e; } int bytesRead = wait(keyFile->read(cipherKey->data(), cipherKey->size(), 0)); if (bytesRead != cipherKey->size()) { - TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize") + TraceEvent(SevError, "InvalidEncryptionKeyFileSize") .detail("ExpectedSize", cipherKey->size()) - .detail("ActualSize", bytesRead); + .detail("ActualSize", bytesRead) + .detail("FileName", encryptionKeyFileName); throw invalid_encryption_key_file(); } ASSERT_EQ(bytesRead, cipherKey->size()); diff --git a/fdbclient/include/fdbclient/BackupContainer.h b/fdbclient/include/fdbclient/BackupContainer.h index bce909ee16..80a3c5a740 100644 --- a/fdbclient/include/fdbclient/BackupContainer.h +++ b/fdbclient/include/fdbclient/BackupContainer.h @@ -316,6 +316,8 @@ public: static std::string lastOpenError; + virtual Future encryptionSetupComplete() const = 0; + // TODO: change the following back to `private` once blob obj access is refactored protected: std::string URL; diff --git a/fdbclient/include/fdbclient/BackupContainerFileSystem.h b/fdbclient/include/fdbclient/BackupContainerFileSystem.h index 4ec6aecc6b..fcb2e4575b 100644 --- a/fdbclient/include/fdbclient/BackupContainerFileSystem.h +++ b/fdbclient/include/fdbclient/BackupContainerFileSystem.h @@ -165,10 +165,14 @@ public: Future writeEncryptionMetadata() override; + // Waits for encryption initialization to complete by reading encryption key file during container opening. + Future encryptionSetupComplete() const override; + protected: + // Returns true if an encryption key file was provided. bool usesEncryption() const; + void setEncryptionKey(Optional const& encryptionKeyFileName); - Future encryptionSetupComplete() const; Future writeEntireFileFallback(const std::string& fileName, const std::string& fileContents); diff --git a/fdbclient/tests/tests_common.sh b/fdbclient/tests/tests_common.sh index a9f7518520..511f239214 100644 --- a/fdbclient/tests/tests_common.sh +++ b/fdbclient/tests/tests_common.sh @@ -176,3 +176,11 @@ function grep_for_severity40 { return 1 fi } + +function test_fdbcli_status_json_for_bkup { + local local_build_dir="${1}" + local local_scratch_dir="${2}" + # Give backup agent time to write status + sleep 5 + "${local_build_dir}"/bin/fdbcli -C "${local_scratch_dir}/loopback_cluster/fdb.cluster" --exec 'status json' | jq '.cluster.layers' +}