[work-serializer] Flush time cache in legacy impl per iteration (#34524)

#34274 is still seeing some headwinds against landing it (I continue to
believe it's the correct fix however).
I'm putting this out as a potential short term mitigation for the load
effects we were seeing, so we can try with folks experiencing problems
and see if it unblocks them.

---------

Co-authored-by: ctiller <ctiller@users.noreply.github.com>

src/core/lib/experiments/experiments.cc

@@ -130,6 +130,10 @@ const char* const description_call_status_override_on_cancellation =
     "with cancellation.";
 const char* const additional_constraints_call_status_override_on_cancellation =
     "{}";
+const char* const description_work_serializer_clears_time_cache =
+    "Have the work serializer clear the time cache when it dispatches work.";
+const char* const additional_constraints_work_serializer_clears_time_cache =
+    "{}";
 #ifdef NDEBUG
 const bool kDefaultForDebugOnly = false;
 #else
@@ -205,6 +209,9 @@ const ExperimentMetadata g_experiment_metadata[] = {
      description_call_status_override_on_cancellation,
      additional_constraints_call_status_override_on_cancellation,
      kDefaultForDebugOnly, true},
+    {"work_serializer_clears_time_cache",
+     description_work_serializer_clears_time_cache,
+     additional_constraints_work_serializer_clears_time_cache, true, true},
 };
 
 }  // namespace grpc_core
@@ -319,6 +326,10 @@ const char* const description_call_status_override_on_cancellation =
     "with cancellation.";
 const char* const additional_constraints_call_status_override_on_cancellation =
     "{}";
+const char* const description_work_serializer_clears_time_cache =
+    "Have the work serializer clear the time cache when it dispatches work.";
+const char* const additional_constraints_work_serializer_clears_time_cache =
+    "{}";
 #ifdef NDEBUG
 const bool kDefaultForDebugOnly = false;
 #else
@@ -394,6 +405,9 @@ const ExperimentMetadata g_experiment_metadata[] = {
      description_call_status_override_on_cancellation,
      additional_constraints_call_status_override_on_cancellation,
      kDefaultForDebugOnly, true},
+    {"work_serializer_clears_time_cache",
+     description_work_serializer_clears_time_cache,
+     additional_constraints_work_serializer_clears_time_cache, true, true},
 };
 
 }  // namespace grpc_core
@@ -508,6 +522,10 @@ const char* const description_call_status_override_on_cancellation =
     "with cancellation.";
 const char* const additional_constraints_call_status_override_on_cancellation =
     "{}";
+const char* const description_work_serializer_clears_time_cache =
+    "Have the work serializer clear the time cache when it dispatches work.";
+const char* const additional_constraints_work_serializer_clears_time_cache =
+    "{}";
 #ifdef NDEBUG
 const bool kDefaultForDebugOnly = false;
 #else
@@ -583,6 +601,9 @@ const ExperimentMetadata g_experiment_metadata[] = {
      description_call_status_override_on_cancellation,
      additional_constraints_call_status_override_on_cancellation,
      kDefaultForDebugOnly, true},
+    {"work_serializer_clears_time_cache",
+     description_work_serializer_clears_time_cache,
+     additional_constraints_work_serializer_clears_time_cache, true, true},
 };
 
 }  // namespace grpc_core

src/core/lib/experiments/experiments.h

@@ -103,6 +103,8 @@ inline bool IsCallStatusOverrideOnCancellationEnabled() {
   return true;
 #endif
 }
+#define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE
+inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; }
 
 #elif defined(GPR_WINDOWS)
 inline bool IsTcpFrameSizeTuningEnabled() { return false; }
@@ -151,6 +153,8 @@ inline bool IsCallStatusOverrideOnCancellationEnabled() {
   return true;
 #endif
 }
+#define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE
+inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; }
 
 #else
 inline bool IsTcpFrameSizeTuningEnabled() { return false; }
@@ -199,6 +203,8 @@ inline bool IsCallStatusOverrideOnCancellationEnabled() {
   return true;
 #endif
 }
+#define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE
+inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; }
 #endif
 
 #else
@@ -232,6 +238,7 @@ enum ExperimentIds {
   kExperimentIdMultiping,
   kExperimentIdRegisteredMethodLookupInTransport,
   kExperimentIdCallStatusOverrideOnCancellation,
+  kExperimentIdWorkSerializerClearsTimeCache,
   kNumExperiments
 };
 #define GRPC_EXPERIMENT_IS_INCLUDED_TCP_FRAME_SIZE_TUNING
@@ -350,6 +357,10 @@ inline bool IsRegisteredMethodLookupInTransportEnabled() {
 inline bool IsCallStatusOverrideOnCancellationEnabled() {
   return IsExperimentEnabled(kExperimentIdCallStatusOverrideOnCancellation);
 }
+#define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE
+inline bool IsWorkSerializerClearsTimeCacheEnabled() {
+  return IsExperimentEnabled(kExperimentIdWorkSerializerClearsTimeCache);
+}
 
 extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];
 

src/core/lib/experiments/experiments.yaml

@@ -222,3 +222,9 @@
   expiry: 2024/01/01
   owner: vigneshbabu@google.com
   test_tags: []
+- name: work_serializer_clears_time_cache
+  description:
+    Have the work serializer clear the time cache when it dispatches work.
+  expiry: 2024/01/01
+  owner: ctiller@google.com
+  test_tags: []

src/core/lib/experiments/rollouts.yaml

@@ -104,3 +104,5 @@
   default: true
 - name: call_status_override_on_cancellation
   default: debug
+- name: work_serializer_clears_time_cache
+  default: true

src/core/lib/gprpp/work_serializer.cc

@@ -262,6 +262,9 @@ void WorkSerializer::LegacyWorkSerializer::DrainQueueOwned() {
     }
     // There is at least one callback on the queue. Pop the callback from the
     // queue and execute it.
+    if (IsWorkSerializerClearsTimeCacheEnabled() && ExecCtx::Get() != nullptr) {
+      ExecCtx::Get()->InvalidateNow();
+    }
     CallbackWrapper* cb_wrapper = nullptr;
     bool empty_unused;
     while ((cb_wrapper = reinterpret_cast<CallbackWrapper*>(