From ca213ccdc029e74f548115dfbb61c65d221e5e42 Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Wed, 21 May 2014 11:44:50 +0200 Subject: [PATCH 1/3] Invalidate key cache on table drop patch by thobbs; reviewed by slebresne for CASSANDRA-6525 --- CHANGES.txt | 1 + src/java/org/apache/cassandra/db/ColumnFamilyStore.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index bd0031e927..432d688a61 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -47,6 +47,7 @@ * Proper null handle for IF with map element access (CASSANDRA-7155) * Improve compaction visibility (CASSANDRA-7242) * Fix 2ndary index queries with DESC clustering order (CASSANDRA-6950) + * Invalid key cache entries on DROP (CASSANDRA-6525) Merged from 1.2: * Add Cloudstack snitch (CASSANDRA-7147) * Update system.peers correctly when relocating tokens (CASSANDRA-7126) diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 3fac6405a2..709935adc1 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -37,6 +37,7 @@ import org.cliffc.high_scale_lib.NonBlockingHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.cache.KeyCacheKey; import org.apache.cassandra.cache.IRowCacheEntry; import org.apache.cassandra.cache.RowCacheKey; import org.apache.cassandra.cache.RowCacheSentinel; @@ -339,6 +340,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean for (RowCacheKey key : CacheService.instance.rowCache.getKeySet()) if (key.cfId == metadata.cfId) invalidateCachedRow(key); + + String ksname = keyspace.getName(); + for (KeyCacheKey key : CacheService.instance.keyCache.getKeySet()) + if (key.getPathInfo().left.equals(ksname) && key.getPathInfo().right.equals(name)) + CacheService.instance.keyCache.remove(key); } /** From 484d2816940cd2eb22d2365fcb376dd27e059e2e Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Wed, 21 May 2014 11:48:21 +0200 Subject: [PATCH 2/3] Fix flapping RecoveryManagerTest patch by benedict; reviewed by jbellis for CASSANDRA-7084 --- CHANGES.txt | 1 + .../apache/cassandra/db/commitlog/CommitLogAllocator.java | 6 ++++-- test/unit/org/apache/cassandra/db/RecoveryManagerTest.java | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 432d688a61..3f8e3d0dca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -48,6 +48,7 @@ * Improve compaction visibility (CASSANDRA-7242) * Fix 2ndary index queries with DESC clustering order (CASSANDRA-6950) * Invalid key cache entries on DROP (CASSANDRA-6525) + * Fix flapping RecoveryManagerTest (CASSANDRA-7084) Merged from 1.2: * Add Cloudstack snitch (CASSANDRA-7147) * Update system.peers correctly when relocating tokens (CASSANDRA-7126) diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogAllocator.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogAllocator.java index 3009a63aed..7ab062bc26 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogAllocator.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogAllocator.java @@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.Uninterruptibles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +37,6 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.io.FSError; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.StorageService; @@ -343,9 +343,11 @@ public class CommitLogAllocator { logger.debug("Closing and clearing existing commit log segments..."); - while (!queue.isEmpty()) + while (StorageService.tasks.getActiveCount() > 0 || !queue.isEmpty()) Thread.yield(); + Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS); + for (CommitLogSegment segment : Iterables.concat(activeSegments, availableSegments)) segment.close(); diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java index c0532112bd..d14d1845b1 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java @@ -23,9 +23,11 @@ import java.util.Date; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.Util; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.commitlog.CommitLog; @@ -35,6 +37,7 @@ import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.Util.column; import static org.apache.cassandra.db.KeyspaceTest.assertColumns; +@RunWith(OrderedJUnit4ClassRunner.class) public class RecoveryManagerTest extends SchemaLoader { @Test From 2635632270289fbbab2f3e3181f20aa98c34a879 Mon Sep 17 00:00:00 2001 From: Sylvain Lebresne Date: Wed, 21 May 2014 17:29:12 +0200 Subject: [PATCH 3/3] Add missing iso8601 patterns for date strings patch by chander; reviewed by slebresne for CASSANDRA-6973 --- CHANGES.txt | 4 + .../apache/cassandra/db/marshal/DateType.java | 41 +--------- .../cassandra/db/marshal/TimeUUIDType.java | 6 +- .../cassandra/db/marshal/TimestampType.java | 43 +---------- .../apache/cassandra/db/marshal/UUIDType.java | 44 +++-------- .../serializers/TimestampSerializer.java | 68 +++++++++++++--- .../serializers/TimestampSerializerTest.java | 77 +++++++++++++++++++ 7 files changed, 155 insertions(+), 128 deletions(-) create mode 100644 test/unit/org/apache/cassandra/serializers/TimestampSerializerTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 3f8e3d0dca..bddb1d1e91 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +2.0.9 + * Add missing iso8601 patterns for date strings (6973) + + 2.0.8 * Always reallocate buffers in HSHA (CASSANDRA-6285) * (Hadoop) support authentication in CqlRecordReader (CASSANDRA-7221) diff --git a/src/java/org/apache/cassandra/db/marshal/DateType.java b/src/java/org/apache/cassandra/db/marshal/DateType.java index 8e28bd48d2..0c97688d3a 100644 --- a/src/java/org/apache/cassandra/db/marshal/DateType.java +++ b/src/java/org/apache/cassandra/db/marshal/DateType.java @@ -38,9 +38,6 @@ public class DateType extends AbstractType public static final DateType instance = new DateType(); - static final String DEFAULT_FORMAT = TimestampSerializer.iso8601Patterns[3]; - static final SimpleDateFormat FORMATTER = new SimpleDateFormat(DEFAULT_FORMAT); - DateType() {} // singleton public int compare(ByteBuffer o1, ByteBuffer o2) @@ -63,43 +60,7 @@ public class DateType extends AbstractType if (source.isEmpty()) return ByteBufferUtil.EMPTY_BYTE_BUFFER; - return ByteBufferUtil.bytes(dateStringToTimestamp(source)); - } - - public static long dateStringToTimestamp(String source) throws MarshalException - { - long millis; - - if (source.toLowerCase().equals("now")) - { - millis = System.currentTimeMillis(); - } - // Milliseconds since epoch? - else if (source.matches("^-?\\d+$")) - { - try - { - millis = Long.parseLong(source); - } - catch (NumberFormatException e) - { - throw new MarshalException(String.format("unable to make long (for date) from: '%s'", source), e); - } - } - // Last chance, attempt to parse as date-time string - else - { - try - { - millis = DateUtils.parseDateStrictly(source, TimestampSerializer.iso8601Patterns).getTime(); - } - catch (ParseException e1) - { - throw new MarshalException(String.format("unable to coerce '%s' to a formatted date (long)", source), e1); - } - } - - return millis; + return ByteBufferUtil.bytes(TimestampSerializer.dateStringToTimestamp(source)); } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java index 51cf47a459..fa82f06e35 100644 --- a/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java +++ b/src/java/org/apache/cassandra/db/marshal/TimeUUIDType.java @@ -25,6 +25,7 @@ import org.apache.cassandra.cql3.CQL3Type; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.TimeUUIDSerializer; +import org.apache.cassandra.serializers.TimestampSerializer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.UUIDGen; @@ -110,9 +111,10 @@ public class TimeUUIDType extends AbstractType if (uuid.version() != 1) throw new MarshalException("TimeUUID supports only version 1 UUIDs"); - } else + } + else { - idBytes = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(DateType.dateStringToTimestamp(source))); + idBytes = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(TimestampSerializer.dateStringToTimestamp(source))); } return idBytes; diff --git a/src/java/org/apache/cassandra/db/marshal/TimestampType.java b/src/java/org/apache/cassandra/db/marshal/TimestampType.java index cf1ea41714..69ead8e93b 100644 --- a/src/java/org/apache/cassandra/db/marshal/TimestampType.java +++ b/src/java/org/apache/cassandra/db/marshal/TimestampType.java @@ -18,9 +18,7 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; -import java.text.ParseException; import java.util.Date; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +27,6 @@ import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.TimestampSerializer; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.commons.lang3.time.DateUtils; /** * Type for date-time values. @@ -44,8 +41,6 @@ public class TimestampType extends AbstractType public static final TimestampType instance = new TimestampType(); - private static final Pattern timestampPattern = Pattern.compile("^-?\\d+$"); - private TimestampType() {} // singleton public int compare(ByteBuffer o1, ByteBuffer o2) @@ -59,43 +54,7 @@ public class TimestampType extends AbstractType if (source.isEmpty()) return ByteBufferUtil.EMPTY_BYTE_BUFFER; - return ByteBufferUtil.bytes(dateStringToTimestamp(source)); - } - - public static long dateStringToTimestamp(String source) throws MarshalException - { - long millis; - - if (source.toLowerCase().equals("now")) - { - millis = System.currentTimeMillis(); - } - // Milliseconds since epoch? - else if (timestampPattern.matcher(source).matches()) - { - try - { - millis = Long.parseLong(source); - } - catch (NumberFormatException e) - { - throw new MarshalException(String.format("unable to make long (for date) from: '%s'", source), e); - } - } - // Last chance, attempt to parse as date-time string - else - { - try - { - millis = DateUtils.parseDateStrictly(source, TimestampSerializer.iso8601Patterns).getTime(); - } - catch (ParseException e1) - { - throw new MarshalException(String.format("unable to coerce '%s' to a formatted date (long)", source), e1); - } - } - - return millis; + return ByteBufferUtil.bytes(TimestampSerializer.dateStringToTimestamp(source)); } @Override diff --git a/src/java/org/apache/cassandra/db/marshal/UUIDType.java b/src/java/org/apache/cassandra/db/marshal/UUIDType.java index 4b0751e261..969ff17f4a 100644 --- a/src/java/org/apache/cassandra/db/marshal/UUIDType.java +++ b/src/java/org/apache/cassandra/db/marshal/UUIDType.java @@ -26,12 +26,11 @@ import org.apache.cassandra.cql3.CQL3Type; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.UUIDSerializer; +import org.apache.cassandra.serializers.TimestampSerializer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.UUIDGen; import org.apache.commons.lang3.time.DateUtils; -import static org.apache.cassandra.serializers.TimestampSerializer.iso8601Patterns; - /** * Compares UUIDs using the following criteria:
* - if count of supplied bytes is less than 16, compare counts
@@ -165,8 +164,6 @@ public class UUIDType extends AbstractType if (source.isEmpty()) return ByteBufferUtil.EMPTY_BYTE_BUFFER; - ByteBuffer idBytes = null; - // ffffffff-ffff-ffff-ffff-ffffffffff if (TimeUUIDType.regexPattern.matcher(source).matches()) { @@ -174,43 +171,22 @@ public class UUIDType extends AbstractType try { uuid = UUID.fromString(source); - idBytes = ByteBuffer.wrap(UUIDGen.decompose(uuid)); + return ByteBuffer.wrap(UUIDGen.decompose(uuid)); } catch (IllegalArgumentException e) { throw new MarshalException(String.format("unable to make UUID from '%s'", source), e); } - } else if (source.toLowerCase().equals("now")) - { - idBytes = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes()); - } - // Milliseconds since epoch? - else if (source.matches("^\\d+$")) - { - try - { - idBytes = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(Long.parseLong(source))); - } - catch (NumberFormatException e) - { - throw new MarshalException(String.format("unable to make version 1 UUID from '%s'", source), e); - } - } - // Last chance, attempt to parse as date-time string - else - { - try - { - long timestamp = DateUtils.parseDate(source, iso8601Patterns).getTime(); - idBytes = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(timestamp)); - } - catch (ParseException e1) - { - throw new MarshalException(String.format("unable to coerce '%s' to version 1 UUID", source), e1); - } } - return idBytes; + try + { + return ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes(TimestampSerializer.dateStringToTimestamp(source))); + } + catch (MarshalException e) + { + throw new MarshalException(String.format("unable to make version 1 UUID from '%s'", source), e); + } } public CQL3Type asCQL3Type() diff --git a/src/java/org/apache/cassandra/serializers/TimestampSerializer.java b/src/java/org/apache/cassandra/serializers/TimestampSerializer.java index f2a40f1316..5cb958600b 100644 --- a/src/java/org/apache/cassandra/serializers/TimestampSerializer.java +++ b/src/java/org/apache/cassandra/serializers/TimestampSerializer.java @@ -21,30 +21,49 @@ import org.apache.cassandra.utils.ByteBufferUtil; import java.nio.ByteBuffer; import java.text.SimpleDateFormat; +import java.text.ParseException; import java.util.Date; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.time.DateUtils; public class TimestampSerializer implements TypeSerializer { - public static final String[] iso8601Patterns = new String[] { + private static final String[] dateStringPatterns = new String[] { "yyyy-MM-dd HH:mm", "yyyy-MM-dd HH:mm:ss", - "yyyy-MM-dd HH:mmZ", - "yyyy-MM-dd HH:mm:ssZ", + "yyyy-MM-dd HH:mmX", + "yyyy-MM-dd HH:mmXX", + "yyyy-MM-dd HH:mmXXX", + "yyyy-MM-dd HH:mm:ssX", + "yyyy-MM-dd HH:mm:ssXX", + "yyyy-MM-dd HH:mm:ssXXX", "yyyy-MM-dd HH:mm:ss.SSS", - "yyyy-MM-dd HH:mm:ss.SSSZ", + "yyyy-MM-dd HH:mm:ss.SSSX", + "yyyy-MM-dd HH:mm:ss.SSSXX", + "yyyy-MM-dd HH:mm:ss.SSSXXX", "yyyy-MM-dd'T'HH:mm", - "yyyy-MM-dd'T'HH:mmZ", + "yyyy-MM-dd'T'HH:mmX", + "yyyy-MM-dd'T'HH:mmXX", + "yyyy-MM-dd'T'HH:mmXXX", "yyyy-MM-dd'T'HH:mm:ss", - "yyyy-MM-dd'T'HH:mm:ssZ", + "yyyy-MM-dd'T'HH:mm:ssX", + "yyyy-MM-dd'T'HH:mm:ssXX", + "yyyy-MM-dd'T'HH:mm:ssXXX", "yyyy-MM-dd'T'HH:mm:ss.SSS", - "yyyy-MM-dd'T'HH:mm:ss.SSSZ", + "yyyy-MM-dd'T'HH:mm:ss.SSSX", + "yyyy-MM-dd'T'HH:mm:ss.SSSXX", + "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", "yyyy-MM-dd", - "yyyy-MM-ddZ" + "yyyy-MM-ddX", + "yyyy-MM-ddXX", + "yyyy-MM-ddXXX" }; - static final String DEFAULT_FORMAT = iso8601Patterns[3]; + private static final String DEFAULT_FORMAT = dateStringPatterns[3]; + private static final Pattern timestampPattern = Pattern.compile("^-?\\d+$"); - static final ThreadLocal FORMATTER = new ThreadLocal() + private static final ThreadLocal FORMATTER = new ThreadLocal() { protected SimpleDateFormat initialValue() { @@ -64,6 +83,35 @@ public class TimestampSerializer implements TypeSerializer return value == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : ByteBufferUtil.bytes(value.getTime()); } + public static long dateStringToTimestamp(String source) throws MarshalException + { + if (source.equalsIgnoreCase("now")) + return System.currentTimeMillis(); + + // Milliseconds since epoch? + if (timestampPattern.matcher(source).matches()) + { + try + { + return Long.parseLong(source); + } + catch (NumberFormatException e) + { + throw new MarshalException(String.format("unable to make long (for date) from: '%s'", source), e); + } + } + + // Last chance, attempt to parse as date-time string + try + { + return DateUtils.parseDateStrictly(source, dateStringPatterns).getTime(); + } + catch (ParseException e1) + { + throw new MarshalException(String.format("unable to coerce '%s' to a formatted date (long)", source), e1); + } + } + public void validate(ByteBuffer bytes) throws MarshalException { if (bytes.remaining() != 8 && bytes.remaining() != 0) diff --git a/test/unit/org/apache/cassandra/serializers/TimestampSerializerTest.java b/test/unit/org/apache/cassandra/serializers/TimestampSerializerTest.java new file mode 100644 index 0000000000..d991845da6 --- /dev/null +++ b/test/unit/org/apache/cassandra/serializers/TimestampSerializerTest.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cassandra.serializers; + +import java.util.List; +import java.util.ArrayList; + +import org.junit.Test; +import static org.junit.Assert.assertTrue; + +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.serializers.TimestampSerializer; + +public class TimestampSerializerTest +{ + private String dates[] = new String[] + { + "2014-04-01", + "2014-04-01+0000", + "2014-04-01 20:30", + "2014-04-01 20:30:35", + "2014-04-01 20:30:35Z", + "2014-04-01 20:30+07", + "2014-04-01 20:30+0700", + "2014-04-01 20:30+07:00", + "2014-04-01 20:30:35+07", + "2014-04-01 20:30:35+0700", + "2014-04-01 20:30:35+07:00", + "2014-04-01 20:30:35.898", + "2014-04-01 20:30:35.898Z", + "2014-04-01 20:30:35.898+07", + "2014-04-01 20:30:35.898+0700", + "2014-04-01 20:30:35.898+07:00", + "2014-04-01T20:30", + "2014-04-01T20:30:25", + "2014-04-01T20:30:35Z", + "2014-04-01T20:30:35+00:00", + "2014-04-01T20:30:35+0700", + "2014-04-01T20:30:35+07:00", + "2014-04-01T20:30:35.898", + "2014-04-01T20:30:35.898+00:00" + }; + + @Test + public void testDateStringToTimestamp() + { + List unparsedDates = new ArrayList<>(); + for (String date: dates) + { + try + { + long millis = TimestampSerializer.dateStringToTimestamp(date); + } + catch (MarshalException e) + { + unparsedDates.add(date); + } + } + assertTrue("Unable to parse: " + unparsedDates, unparsedDates.isEmpty()); + } +}