diff --git a/build.xml b/build.xml index 2b93e7d361..f801791383 100644 --- a/build.xml +++ b/build.xml @@ -238,8 +238,10 @@ sourcesFilesetId="test-dependency-sources" cacheDependencyRefs="true" dependencyRefsBuildFile="${build.dir}/test-dependencies.xml"> - - + + + + diff --git a/test/distributed/README.txt b/test/distributed/README.txt index 44e1ab9107..651038b77c 100644 --- a/test/distributed/README.txt +++ b/test/distributed/README.txt @@ -26,7 +26,8 @@ configuration is located at: An example EC2/S3 whirr configuration would be: ############################################### -whirr.provider=ec2 +whirr.cluster-user=[username] +whirr.provider=aws-ec2 whirr.location-id=us-west-1 whirr.image-id=us-west-1/ami-16f3a253 whirr.hardware-id=m1.large @@ -35,7 +36,7 @@ whirr.credential=[EC2 Secret Access Key] whirr.private-key-file=${sys:user.home}/.ssh/id_rsa whirr.public-key-file=${sys:user.home}/.ssh/id_rsa.pub whirr.run-url-base=http://hoodidge.net/scripts/ -whirr.blobstore.provider=s3 +whirr.blobstore.provider=aws-s3 whirr.blobstore.container=cassandratests ############################################### diff --git a/test/distributed/org/apache/cassandra/CassandraServiceController.java b/test/distributed/org/apache/cassandra/CassandraServiceController.java index 0684cf7e7e..3ffcb3fe17 100644 --- a/test/distributed/org/apache/cassandra/CassandraServiceController.java +++ b/test/distributed/org/apache/cassandra/CassandraServiceController.java @@ -18,52 +18,41 @@ package org.apache.cassandra; -import java.io.IOException; import java.net.InetAddress; import java.net.URI; import java.util.*; -import org.apache.cassandra.thrift.Cassandra; -import org.apache.cassandra.thrift.TokenRange; -import org.apache.cassandra.utils.KeyPair; -import org.apache.cassandra.utils.BlobUtils; -import org.apache.cassandra.utils.Pair; +import com.google.common.base.Predicate; -import org.apache.commons.configuration.Configuration; +import org.apache.cassandra.thrift.Cassandra; +import org.apache.cassandra.utils.BlobUtils; +import org.apache.cassandra.utils.KeyPair; +import org.apache.cassandra.utils.Pair; import org.apache.commons.configuration.CompositeConfiguration; import org.apache.commons.configuration.PropertiesConfiguration; - import org.apache.thrift.TException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.transport.*; - -import org.apache.whirr.service.Cluster; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.whirr.service.*; import org.apache.whirr.service.Cluster.Instance; -import org.apache.whirr.service.ClusterSpec; -import org.apache.whirr.service.ComputeServiceContextBuilder; -import org.apache.whirr.service.Service; -import org.apache.whirr.service.ServiceFactory; -import org.apache.whirr.service.cassandra.CassandraService; import org.apache.whirr.service.cassandra.CassandraClusterActionHandler; import org.apache.whirr.service.jclouds.RunUrlStatement; import org.jclouds.blobstore.domain.BlobMetadata; - import org.jclouds.compute.ComputeService; +import org.jclouds.compute.domain.ExecResponse; import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.compute.options.RunScriptOptions; import org.jclouds.domain.Credentials; -import org.jclouds.io.Payload; import org.jclouds.scriptbuilder.domain.OsFamily; -import org.jclouds.ssh.ExecResponse; -import static org.jclouds.io.Payloads.newStringPayload; - -import com.google.common.base.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.assertThat; +import static org.jclouds.io.Payloads.newStringPayload; public class CassandraServiceController { @@ -84,7 +73,7 @@ public class CassandraServiceController private boolean running; private ClusterSpec clusterSpec; - private CassandraService service; + private Service service; private Cluster cluster; private ComputeService computeService; private Credentials credentials; @@ -96,8 +85,7 @@ public class CassandraServiceController { } - public Cassandra.Client createClient(InetAddress addr) - throws TTransportException, TException + public Cassandra.Client createClient(InetAddress addr) throws TException { TTransport transport = new TSocket( addr.getHostAddress(), @@ -170,10 +158,10 @@ public class CassandraServiceController tarball = blob.left; config.setProperty(CassandraClusterActionHandler.BIN_TARBALL, blob.right.toURL().toString()); // TODO: parse the CassandraVersion property file instead - config.setProperty(CassandraClusterActionHandler.MAJOR_VERSION, "0.7"); + config.setProperty(CassandraClusterActionHandler.MAJOR_VERSION, "0.8"); } - service = (CassandraService)new ServiceFactory().create(clusterSpec.getServiceName()); + service = new ServiceFactory().create(clusterSpec.getServiceName()); cluster = service.launchCluster(clusterSpec); computeService = ComputeServiceContextBuilder.build(clusterSpec).getComputeService(); hosts = new ArrayList(); diff --git a/test/distributed/org/apache/cassandra/utils/BlobUtils.java b/test/distributed/org/apache/cassandra/utils/BlobUtils.java index 94474cf086..ddcbfa5fc1 100644 --- a/test/distributed/org/apache/cassandra/utils/BlobUtils.java +++ b/test/distributed/org/apache/cassandra/utils/BlobUtils.java @@ -19,24 +19,26 @@ package org.apache.cassandra.utils; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.net.URI; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import com.google.common.io.Files; import org.apache.commons.configuration.Configuration; import org.apache.whirr.service.ClusterSpec; -import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContextFactory; -import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.InputStreamMap; +import org.jclouds.blobstore.domain.BlobMetadata; -import org.jclouds.aws.s3.S3Client; -import org.jclouds.aws.s3.S3AsyncClient; -import org.jclouds.aws.s3.domain.AccessControlList; -import org.jclouds.aws.s3.domain.CannedAccessPolicy; - -import org.jclouds.rest.RestContext; +import org.jclouds.s3.S3AsyncClient; +import org.jclouds.s3.S3Client; +import org.jclouds.s3.domain.AccessControlList; +import org.jclouds.s3.domain.CannedAccessPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,24 +79,63 @@ public final class BlobUtils File file = new File(filename); String container = getContainer(config); String provider = getProvider(config); + + // blob name and checksum of the file String blobName = System.nanoTime() + "/" + file.getName(); + String blobNameChecksum = blobName + ".md5"; + BlobStoreContext context = getContext(config, spec); + + File checksumFile; + + try + { + checksumFile = File.createTempFile("dtchecksum", "md5"); + checksumFile.deleteOnExit(); + + FileWriter checksumWriter = new FileWriter(checksumFile); + + String checksum = FBUtilities.bytesToHex(Files.getDigest(file, MessageDigest.getInstance("MD5"))); + + checksumWriter.write(String.format("%s %s", checksum, file.getName())); + checksumWriter.close(); + } + catch (IOException e) + { + throw new RuntimeException("Can't create a checksum of the file: " + filename); + } + catch (NoSuchAlgorithmException e) + { + throw new RuntimeException(e.getMessage()); + } + try { InputStreamMap map = context.createInputStreamMap(container); + map.putFile(blobName, file); + map.putFile(blobNameChecksum, checksumFile); + // TODO: magic! in order to expose the blob as public, we need to dive into provider specific APIs // the hope is that permissions are encapsulated in jclouds in the future - if (provider.equals("s3")) + if (provider.contains("s3")) { S3Client sss = context.getProviderSpecificContext().getApi(); String ownerId = sss.getObjectACL(container, blobName).getOwner().getId(); + sss.putObjectACL(container, blobName, AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PUBLIC_READ, ownerId)); + + sss.putObjectACL(container, + blobNameChecksum, + AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PUBLIC_READ, ownerId)); } else + { LOG.warn(provider + " may not be properly supported for tarball transfer."); + } + // resolve the full URI of the blob (see http://code.google.com/p/jclouds/issues/detail?id=431) BlobMetadata blob = context.getBlobStore().blobMetadata(container, blobName); URI uri = context.getProviderSpecificContext().getEndpoint().resolve("/" + container + "/" + blob.getName()); diff --git a/test/resources/whirr-default.properties b/test/resources/whirr-default.properties index fc33c48122..5f2066080e 100644 --- a/test/resources/whirr-default.properties +++ b/test/resources/whirr-default.properties @@ -17,5 +17,6 @@ # whirr.service-name=cassandra whirr.cluster-name=cassandra_test +whirr.cluster-user=app whirr.instance-templates=4 cassandra -whirr.version=0.3.0-incubating-SNAPSHOT +whirr.version=0.4.0-incubating diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/tools/sstabledebug/src/org/apache/cassandra/sstabledebug/SSTableExport.java similarity index 99% rename from src/java/org/apache/cassandra/tools/SSTableExport.java rename to tools/sstabledebug/src/org/apache/cassandra/sstabledebug/SSTableExport.java index e5aca7de9e..b79c0f1502 100644 --- a/src/java/org/apache/cassandra/tools/SSTableExport.java +++ b/tools/sstabledebug/src/org/apache/cassandra/sstabledebug/SSTableExport.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.tools; +package org.apache.cassandra.sstabledebug; import java.io.File; import java.io.IOException; diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/tools/sstabledebug/src/org/apache/cassandra/sstabledebug/SSTableImport.java similarity index 99% rename from src/java/org/apache/cassandra/tools/SSTableImport.java rename to tools/sstabledebug/src/org/apache/cassandra/sstabledebug/SSTableImport.java index 1b53563f14..d3f70d0473 100644 --- a/src/java/org/apache/cassandra/tools/SSTableImport.java +++ b/tools/sstabledebug/src/org/apache/cassandra/sstabledebug/SSTableImport.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.tools; +package org.apache.cassandra.sstabledebug; import java.io.File; import java.io.IOException;