From 742e5baf6311a1141b42bec0b3c3dc2ff19fa376 Mon Sep 17 00:00:00 2001 From: Vijay Parthasarathy Date: Sat, 14 Sep 2013 16:05:54 -0700 Subject: [PATCH 1/2] Fix CommitLogReplayer date time issue patch by Vijay ; reviewed by jbellis for CASSANDRA-5909 --- conf/commitlog_archiving.properties | 7 +++-- .../db/commitlog/CommitLogArchiver.java | 12 +++++++- .../db/commitlog/CommitLogReplayer.java | 2 +- test/conf/commitlog_archiving.properties | 19 ++++++++++++ .../cassandra/db/RecoveryManagerTest.java | 29 +++++++++++++++++++ 5 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 test/conf/commitlog_archiving.properties diff --git a/conf/commitlog_archiving.properties b/conf/commitlog_archiving.properties index 23adaeb9ef..b19cfed037 100644 --- a/conf/commitlog_archiving.properties +++ b/conf/commitlog_archiving.properties @@ -43,8 +43,8 @@ restore_command= # Directory to scan the recovery files in. restore_directories= -# Restore mutations created up to and including this timestamp. -# Format: 2012-04-31 20:43:12 +# Restore mutations created up to and including this timestamp in GMT. +# Format: yyyy:MM:dd HH:mm:ss (2012:04:31 20:43:12) # # Note! Recovery will stop when the first client-supplied timestamp # greater than this time is encountered. Since the order Cassandra @@ -52,3 +52,6 @@ restore_directories= # this may leave some mutations with timestamps earlier than the # point-in-time unrecovered. restore_point_in_time= + +# precision of the timestamp used in the inserts (MILLISECONDS, MICROSECONDS, ...) +precision=MILLISECONDS \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java index e9d850ae0d..78026a4019 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java @@ -27,6 +27,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Map; import java.util.Properties; +import java.util.TimeZone; import java.util.concurrent.*; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; @@ -43,12 +44,19 @@ import com.google.common.base.Strings; public class CommitLogArchiver { private static final Logger logger = LoggerFactory.getLogger(CommitLogArchiver.class); + public static final SimpleDateFormat format = new SimpleDateFormat("yyyy:MM:dd HH:mm:ss"); + static + { + format.setTimeZone(TimeZone.getTimeZone("GMT")); + } + public final Map> archivePending = new ConcurrentHashMap>(); public final ExecutorService executor = new JMXEnabledThreadPoolExecutor("commitlog_archiver"); private final String archiveCommand; private final String restoreCommand; private final String restoreDirectories; public final long restorePointInTime; + public final TimeUnit precision; public CommitLogArchiver() { @@ -65,6 +73,7 @@ public class CommitLogArchiver restoreCommand = null; restoreDirectories = null; restorePointInTime = Long.MAX_VALUE; + precision = TimeUnit.MILLISECONDS; } else { @@ -73,9 +82,10 @@ public class CommitLogArchiver restoreCommand = commitlog_commands.getProperty("restore_command"); restoreDirectories = commitlog_commands.getProperty("restore_directories"); String targetTime = commitlog_commands.getProperty("restore_point_in_time"); + precision = TimeUnit.valueOf(commitlog_commands.getProperty("precision", "MILLISECONDS")); try { - restorePointInTime = Strings.isNullOrEmpty(targetTime) ? Long.MAX_VALUE : new SimpleDateFormat("yyyy:MM:dd HH:mm:ss").parse(targetTime).getTime(); + restorePointInTime = Strings.isNullOrEmpty(targetTime) ? Long.MAX_VALUE : format.parse(targetTime).getTime(); } catch (ParseException e) { diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java index 6b401fb07c..796ab5b53e 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java @@ -281,7 +281,7 @@ public class CommitLogReplayer for (ColumnFamily families : frm.getColumnFamilies()) { - if (families.maxTimestamp() > restoreTarget) + if (CommitLog.instance.archiver.precision.toMillis(families.maxTimestamp()) > restoreTarget) return true; } return false; diff --git a/test/conf/commitlog_archiving.properties b/test/conf/commitlog_archiving.properties new file mode 100644 index 0000000000..aaf6bd1e12 --- /dev/null +++ b/test/conf/commitlog_archiving.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +restore_point_in_time=2112:12:12 12:12:12 +precision=MICROSECONDS \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java index 68c0b37067..9c17d8043f 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java @@ -19,13 +19,17 @@ package org.apache.cassandra.db; import java.io.IOException; +import java.util.Date; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import org.apache.cassandra.Util; +import org.junit.Assert; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.db.commitlog.CommitLogArchiver; import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.Util.column; @@ -101,4 +105,29 @@ public class RecoveryManagerTest extends SchemaLoader assert c != null; assert ((CounterColumn)c).total() == 10L; } + + @Test + public void testRecoverPIT() throws Exception + { + Date date = CommitLogArchiver.format.parse("2112:12:12 12:12:12"); + long timeMS = date.getTime() - 5000; + + Table keyspace1 = Table.open("Keyspace1"); + DecoratedKey dk = Util.dk("dkey"); + for (int i = 0; i < 10; ++i) + { + long ts = TimeUnit.MILLISECONDS.toMicros(timeMS + (i * 1000)); + ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1"); + cf.addColumn(column("name-" + i, "value", ts)); + RowMutation rm = new RowMutation("Keyspace1", dk.key); + rm.add(cf); + rm.apply(); + } + keyspace1.getColumnFamilyStore("Standard1").clearUnsafe(); + CommitLog.instance.resetUnsafe(); // disassociate segments from live CL + CommitLog.instance.recover(); + + ColumnFamily cf = Util.getColumnFamily(keyspace1, dk, "Standard1"); + Assert.assertEquals(6, cf.getColumnCount()); + } } From 3bba32c409e9e6632063a5b5086e8ceb110dab33 Mon Sep 17 00:00:00 2001 From: Vijay Parthasarathy Date: Sat, 14 Sep 2013 17:45:00 -0700 Subject: [PATCH 2/2] Additional NEWS and test fixes for 2.0 (part of CASSANDRA-5909) --- NEWS.txt | 2 ++ conf/commitlog_archiving.properties | 2 +- .../apache/cassandra/db/commitlog/CommitLogArchiver.java | 4 ++-- test/unit/org/apache/cassandra/db/RecoveryManagerTest.java | 7 +++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index 4e11bf21fc..68da5978c7 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -76,6 +76,8 @@ Operations use DESCRIBE FULL SCHEMA if you need the schema of system_* keyspaces. - CQL2 has been deprecated, and will be removed entirely in 2.2. See CASSANDRA-5918 for details. + - Commit log archiver now assumes the client time stamp to be in microsecond + precision, during restore. Please refer to commitlog_archiving.properties. Features diff --git a/conf/commitlog_archiving.properties b/conf/commitlog_archiving.properties index b19cfed037..0687cd60b4 100644 --- a/conf/commitlog_archiving.properties +++ b/conf/commitlog_archiving.properties @@ -54,4 +54,4 @@ restore_directories= restore_point_in_time= # precision of the timestamp used in the inserts (MILLISECONDS, MICROSECONDS, ...) -precision=MILLISECONDS \ No newline at end of file +precision=MICROSECONDS \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java index 78026a4019..f020182f9b 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java @@ -73,7 +73,7 @@ public class CommitLogArchiver restoreCommand = null; restoreDirectories = null; restorePointInTime = Long.MAX_VALUE; - precision = TimeUnit.MILLISECONDS; + precision = TimeUnit.MICROSECONDS; } else { @@ -82,7 +82,7 @@ public class CommitLogArchiver restoreCommand = commitlog_commands.getProperty("restore_command"); restoreDirectories = commitlog_commands.getProperty("restore_directories"); String targetTime = commitlog_commands.getProperty("restore_point_in_time"); - precision = TimeUnit.valueOf(commitlog_commands.getProperty("precision", "MILLISECONDS")); + precision = TimeUnit.valueOf(commitlog_commands.getProperty("precision", "MICROSECONDS")); try { restorePointInTime = Strings.isNullOrEmpty(targetTime) ? Long.MAX_VALUE : format.parse(targetTime).getTime(); diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java index 4cb9e4eb89..c0532112bd 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java @@ -109,15 +109,14 @@ public class RecoveryManagerTest extends SchemaLoader Date date = CommitLogArchiver.format.parse("2112:12:12 12:12:12"); long timeMS = date.getTime() - 5000; - Table keyspace1 = Table.open("Keyspace1"); + Keyspace keyspace1 = Keyspace.open("Keyspace1"); DecoratedKey dk = Util.dk("dkey"); for (int i = 0; i < 10; ++i) { long ts = TimeUnit.MILLISECONDS.toMicros(timeMS + (i * 1000)); - ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1"); + ColumnFamily cf = TreeMapBackedSortedColumns.factory.create("Keyspace1", "Standard1"); cf.addColumn(column("name-" + i, "value", ts)); - RowMutation rm = new RowMutation("Keyspace1", dk.key); - rm.add(cf); + RowMutation rm = new RowMutation("Keyspace1", dk.key, cf); rm.apply(); } keyspace1.getColumnFamilyStore("Standard1").clearUnsafe();