Encryption changes in status json (#12657)

* EncryptionBackup: Log encryption key file access failures at SevError severity (#12629)

* Change the error to Sev40

* Fix formatting error

* Display Encryption Key Info in status json (#12649)

* Encryption in json

* Addressed comments

* Use unordered_set

* Addressed comments for ctest

fdbbackup/backup.actor.cpp

@@ -50,6 +50,7 @@
 #include "fdbclient/S3BlobStore.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/json_spirit/json_spirit_writer_template.h"
+#include "fdbclient/BackupContainer.h"
 
 #include "flow/Platform.h"
 
@@ -1612,6 +1613,32 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 		wait(waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) &&
 		     waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused));
 
+		state std::vector<Future<Void>> encryptionSetupResults;
+		state std::vector<int> encryptionContainerIndices;
+
+		for (int i = 0; i < tagContainers.size(); i++) {
+			if (tagContainers[i].get()->getEncryptionKeyFileName().present()) {
+				encryptionSetupResults.push_back(tagContainers[i].get()->encryptionSetupComplete());
+				encryptionContainerIndices.push_back(i);
+			}
+		}
+		wait(waitForAllReady(encryptionSetupResults));
+		json_spirit::mArray keysArr;
+		std::unordered_set<std::string> seenKeyPaths;
+		for (int j = 0; j < encryptionContainerIndices.size() && j < 1e6; j++) {
+			int i = encryptionContainerIndices[j];
+			std::string keyPath = tagContainers[i].get()->getEncryptionKeyFileName().get();
+
+			if (seenKeyPaths.find(keyPath) == seenKeyPaths.end()) {
+				seenKeyPaths.insert(keyPath);
+				json_spirit::mObject keyObj;
+				keyObj["path"] = tagContainers[i].get()->getEncryptionKeyFileName().get();
+				keyObj["success"] = !encryptionSetupResults[j].isError();
+				keysArr.push_back(keyObj);
+			}
+		}
+		o.create("encryption_keys") = keysArr;
+
 		JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest");
 		layerRoot.create("tags.timestamp") = now();
 		layerRoot.create("total_workers.$sum") =
@@ -1640,7 +1667,11 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
 			tagRoot.create("range_bytes_written") = tagRangeBytes[j].get();
 			tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get();
 			tagRoot.create("mutation_stream_id") = backupTagUids[j].toString();
-
+			tagRoot.create("file_level_encryption") =
+			    tagContainers[j].get()->getEncryptionKeyFileName().present() ? true : false;
+			if (tagContainers[j].get()->getEncryptionKeyFileName().present()) {
+				tagRoot.create("encryption_key_file") = tagContainers[j].get()->getEncryptionKeyFileName().get();
+			}
 			j++;
 		}
 	} else if (exe == ProgramExe::DR_AGENT) {

fdbbackup/tests/dir_backup_test.sh

@@ -104,6 +104,9 @@ function test_dir_backup_and_restore {
     err "Failed backup"
     return 1
   fi
+
+  test_fdbcli_status_json_for_bkup "${local_build_dir}" "${scratch_dir}"
+
   log "Clear fdb data"
   if ! clear_data "${local_build_dir}" "${scratch_dir}"; then
     err "Failed clear data in fdb"

fdbbackup/tests/s3_backup_test.sh

@@ -132,6 +132,9 @@ function test_s3_backup_and_restore {
     err "Failed backup"
     return 1
   fi
+
+  test_fdbcli_status_json_for_bkup "${local_build_dir}" "${local_scratch_dir}"
+
   log "Clear fdb data"
   if ! clear_data "${local_build_dir}" "${local_scratch_dir}"; then
     err "Failed clear data in fdb"

fdbclient/BackupContainerFileSystem.actor.cpp

@@ -1307,16 +1307,15 @@ public:
 			    0400));
 			keyFile = _keyFile;
 		} catch (Error& e) {
-			TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile")
-			    .error(e)
-			    .detail("FileName", encryptionKeyFileName);
+			TraceEvent(SevError, "FailedToOpenEncryptionKeyFile").error(e).detail("FileName", encryptionKeyFileName);
 			throw e;
 		}
 		int bytesRead = wait(keyFile->read(cipherKey->data(), cipherKey->size(), 0));
 		if (bytesRead != cipherKey->size()) {
-			TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize")
+			TraceEvent(SevError, "InvalidEncryptionKeyFileSize")
 			    .detail("ExpectedSize", cipherKey->size())
-			    .detail("ActualSize", bytesRead);
+			    .detail("ActualSize", bytesRead)
+			    .detail("FileName", encryptionKeyFileName);
 			throw invalid_encryption_key_file();
 		}
 		ASSERT_EQ(bytesRead, cipherKey->size());

fdbclient/include/fdbclient/BackupContainer.h

@@ -316,6 +316,8 @@ public:
 
 	static std::string lastOpenError;
 
+	virtual Future<Void> encryptionSetupComplete() const = 0;
+
 	// TODO: change the following back to `private` once blob obj access is refactored
 protected:
 	std::string URL;

fdbclient/include/fdbclient/BackupContainerFileSystem.h

@@ -165,10 +165,14 @@ public:
 
 	Future<Void> writeEncryptionMetadata() override;
 
+	// Waits for encryption initialization to complete by reading encryption key file during container opening.
+	Future<Void> encryptionSetupComplete() const override;
+
 protected:
+	// Returns true if an encryption key file was provided.
 	bool usesEncryption() const;
+
 	void setEncryptionKey(Optional<std::string> const& encryptionKeyFileName);
-	Future<Void> encryptionSetupComplete() const;
 
 	Future<Void> writeEntireFileFallback(const std::string& fileName, const std::string& fileContents);
 

fdbclient/tests/tests_common.sh

@@ -176,3 +176,11 @@ function grep_for_severity40 {
     return 1
   fi
 }
+
+function test_fdbcli_status_json_for_bkup {
+  local local_build_dir="${1}"
+  local local_scratch_dir="${2}"
+  # Give backup agent time to write status
+  sleep 5
+  "${local_build_dir}"/bin/fdbcli -C "${local_scratch_dir}/loopback_cluster/fdb.cluster" --exec 'status json' | jq '.cluster.layers'
+}