224 lines
7.4 KiB
Python
224 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import shutil
|
|
import stat
|
|
from urllib import request
|
|
import hashlib
|
|
|
|
from local_cluster import random_secret_string
|
|
|
|
SUPPORTED_PLATFORMS = ["x86_64", "aarch64"]
|
|
SUPPORTED_VERSIONS = [
|
|
"7.2.0",
|
|
"7.1.9",
|
|
"7.1.8",
|
|
"7.1.7",
|
|
"7.1.6",
|
|
"7.1.5",
|
|
"7.1.4",
|
|
"7.1.3",
|
|
"7.1.2",
|
|
"7.1.1",
|
|
"7.1.0",
|
|
"7.0.0",
|
|
"6.3.24",
|
|
"6.3.23",
|
|
"6.3.22",
|
|
"6.3.18",
|
|
"6.3.17",
|
|
"6.3.16",
|
|
"6.3.15",
|
|
"6.3.13",
|
|
"6.3.12",
|
|
"6.3.9",
|
|
"6.2.30",
|
|
"6.2.29",
|
|
"6.2.28",
|
|
"6.2.27",
|
|
"6.2.26",
|
|
"6.2.25",
|
|
"6.2.24",
|
|
"6.2.23",
|
|
"6.2.22",
|
|
"6.2.21",
|
|
"6.2.20",
|
|
"6.2.19",
|
|
"6.2.18",
|
|
"6.2.17",
|
|
"6.2.16",
|
|
"6.2.15",
|
|
"6.2.10",
|
|
"6.1.13",
|
|
"6.1.12",
|
|
"6.1.11",
|
|
"6.1.10",
|
|
"6.0.18",
|
|
"6.0.17",
|
|
"6.0.16",
|
|
"6.0.15",
|
|
"6.0.14",
|
|
"5.2.8",
|
|
"5.2.7",
|
|
"5.1.7",
|
|
"5.1.6",
|
|
]
|
|
FDB_DOWNLOAD_ROOT = "https://github.com/apple/foundationdb/releases/download/"
|
|
LOCAL_OLD_BINARY_REPO = "/opt/foundationdb/old/"
|
|
CURRENT_VERSION = "7.2.0"
|
|
MAX_DOWNLOAD_ATTEMPTS = 5
|
|
|
|
|
|
def make_executable_path(path):
|
|
st = os.stat(path)
|
|
os.chmod(path, st.st_mode | stat.S_IEXEC)
|
|
|
|
|
|
def compute_sha256(filename):
|
|
hash_function = hashlib.sha256()
|
|
with open(filename, "rb") as f:
|
|
while True:
|
|
data = f.read(128 * 1024)
|
|
if not data:
|
|
break
|
|
hash_function.update(data)
|
|
|
|
return hash_function.hexdigest()
|
|
|
|
|
|
def read_to_str(filename):
|
|
with open(filename, "r") as f:
|
|
return f.read()
|
|
|
|
|
|
class FdbBinaryDownloader:
|
|
def __init__(
|
|
self,
|
|
build_dir
|
|
):
|
|
self.build_dir = Path(build_dir).resolve()
|
|
assert self.build_dir.exists(), "{} does not exist".format(build_dir)
|
|
assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir)
|
|
self.platform = platform.machine()
|
|
assert self.platform in SUPPORTED_PLATFORMS, "Unsupported platform {}".format(
|
|
self.platform
|
|
)
|
|
self.tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
|
|
self.tmp_dir.mkdir(parents=True)
|
|
self.download_dir = self.build_dir.joinpath("tmp", "old_binaries")
|
|
self.local_binary_repo = Path(LOCAL_OLD_BINARY_REPO)
|
|
if not self.local_binary_repo.exists():
|
|
self.local_binary_repo = None
|
|
|
|
# Check if the binaries for the given version are available in the local old binaries repository
|
|
def version_in_local_repo(self, version):
|
|
return (self.local_binary_repo is not None) and (self.local_binary_repo.joinpath(version).exists())
|
|
|
|
def binary_path(self, version, bin_name):
|
|
if version == CURRENT_VERSION:
|
|
return self.build_dir.joinpath("bin", bin_name)
|
|
elif self.version_in_local_repo(version):
|
|
return self.local_binary_repo.joinpath(version, "bin", "{}-{}".format(bin_name, version))
|
|
else:
|
|
return self.download_dir.joinpath(version, bin_name)
|
|
|
|
def lib_dir(self, version):
|
|
if version == CURRENT_VERSION:
|
|
return self.build_dir.joinpath("lib")
|
|
else:
|
|
return self.download_dir.joinpath(version)
|
|
|
|
def lib_path(self, version):
|
|
return self.lib_dir(version).joinpath("libfdb_c.so")
|
|
|
|
# Download an old binary of a given version from a remote repository
|
|
def download_old_binary(
|
|
self, version, target_bin_name, remote_bin_name, make_executable
|
|
):
|
|
local_file = self.download_dir.joinpath(version, target_bin_name)
|
|
if local_file.exists():
|
|
return
|
|
|
|
# Download to a temporary file and then replace the target file atomically
|
|
# to avoid consistency errors in case of multiple tests are downloading the
|
|
# same file in parallel
|
|
local_file_tmp = Path("{}.{}".format(str(local_file), random_secret_string(8)))
|
|
self.download_dir.joinpath(version).mkdir(parents=True, exist_ok=True)
|
|
remote_file = "{}{}/{}".format(FDB_DOWNLOAD_ROOT, version, remote_bin_name)
|
|
remote_sha256 = "{}.sha256".format(remote_file)
|
|
local_sha256 = Path("{}.sha256".format(local_file_tmp))
|
|
|
|
for attempt_cnt in range(MAX_DOWNLOAD_ATTEMPTS + 1):
|
|
if attempt_cnt == MAX_DOWNLOAD_ATTEMPTS:
|
|
assert False, "Failed to download {} after {} attempts".format(
|
|
local_file_tmp, MAX_DOWNLOAD_ATTEMPTS
|
|
)
|
|
try:
|
|
print("Downloading '{}' to '{}'...".format(remote_file, local_file_tmp))
|
|
request.urlretrieve(remote_file, local_file_tmp)
|
|
print("Downloading '{}' to '{}'...".format(remote_sha256, local_sha256))
|
|
request.urlretrieve(remote_sha256, local_sha256)
|
|
print("Download complete")
|
|
except Exception as e:
|
|
print("Retrying on error:", e)
|
|
continue
|
|
|
|
assert local_file_tmp.exists(), "{} does not exist".format(local_file_tmp)
|
|
assert local_sha256.exists(), "{} does not exist".format(local_sha256)
|
|
expected_checksum = read_to_str(local_sha256)
|
|
actual_checkum = compute_sha256(local_file_tmp)
|
|
if expected_checksum == actual_checkum:
|
|
print("Checksum OK")
|
|
break
|
|
print(
|
|
"Checksum mismatch. Expected: {} Actual: {}".format(
|
|
expected_checksum, actual_checkum
|
|
)
|
|
)
|
|
|
|
os.rename(local_file_tmp, local_file)
|
|
os.remove(local_sha256)
|
|
|
|
if make_executable:
|
|
make_executable_path(local_file)
|
|
|
|
# Copy a client library file from the local old binaries repository
|
|
# The file needs to be renamed to libfdb_c.so, because it is loaded with this name by fdbcli
|
|
def copy_clientlib_from_local_repo(self, version):
|
|
dest_lib_file = self.download_dir.joinpath(version, "libfdb_c.so")
|
|
if dest_lib_file.exists():
|
|
return
|
|
# Avoid race conditions in case of parallel test execution by first copying to a temporary file
|
|
# and then renaming it atomically
|
|
dest_file_tmp = Path("{}.{}".format(str(dest_lib_file), random_secret_string(8)))
|
|
src_lib_file = self.local_binary_repo.joinpath(version, "lib", "libfdb_c-{}.so".format(version))
|
|
assert src_lib_file.exists(), "Missing file {} in the local old binaries repository".format(src_lib_file)
|
|
self.download_dir.joinpath(version).mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(src_lib_file, dest_file_tmp)
|
|
os.rename(dest_file_tmp, dest_lib_file)
|
|
assert dest_lib_file.exists(), "{} does not exist".format(dest_lib_file)
|
|
|
|
# Download all old binaries required for testing the specified upgrade path
|
|
def download_old_binaries(self, version):
|
|
if version == CURRENT_VERSION:
|
|
return
|
|
|
|
if self.version_in_local_repo(version):
|
|
self.copy_clientlib_from_local_repo(version)
|
|
return
|
|
|
|
self.download_old_binary(
|
|
version, "fdbserver", "fdbserver.{}".format(self.platform), True
|
|
)
|
|
self.download_old_binary(
|
|
version, "fdbmonitor", "fdbmonitor.{}".format(self.platform), True
|
|
)
|
|
self.download_old_binary(
|
|
version, "fdbcli", "fdbcli.{}".format(self.platform), True
|
|
)
|
|
self.download_old_binary(
|
|
version, "libfdb_c.so", "libfdb_c.{}.so".format(self.platform), False
|
|
)
|