Introduce experiment to MADV_NOHUGEPAGE HugeRegion managed memory.

tcmalloc/experiment_config.h

@@ -28,6 +28,7 @@ enum class Experiment : int {
   TCMALLOC_PER_CPU_CACHE_SIZE_1MB,  // TODO: b/514747820 - Complete experiment.
   TCMALLOC_PGHO_EXPERIMENT,  // TODO: b/460486507 - Complete experiment.
   TCMALLOC_REUSE_SIZE_CLASSES_ABLATION,  // TODO: b/524296402 - Complete experiment.
+  TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS,  // TODO: b/527907199 - Complete experiment.
   TEST_ONLY_L3_AWARE,  // TODO: b/239977380 - Complete experiment.
   TEST_ONLY_MM_VCPU,  // TODO: b/245776120 - Complete experiment.
   TEST_ONLY_TCMALLOC_HEAP_PARTITIONING,  // TODO: b/446814339 - Complete experiment.
@@ -58,6 +59,7 @@ inline constexpr ExperimentConfig experiments[] = {
     {Experiment::TCMALLOC_PER_CPU_CACHE_SIZE_1MB, "TCMALLOC_PER_CPU_CACHE_SIZE_1MB"},
     {Experiment::TCMALLOC_PGHO_EXPERIMENT, "TCMALLOC_PGHO_EXPERIMENT"},
     {Experiment::TCMALLOC_REUSE_SIZE_CLASSES_ABLATION, "TCMALLOC_REUSE_SIZE_CLASSES_ABLATION"},
+    {Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS, "TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS"},
     {Experiment::TEST_ONLY_L3_AWARE, "TEST_ONLY_L3_AWARE"},
     {Experiment::TEST_ONLY_MM_VCPU, "TEST_ONLY_MM_VCPU"},
     {Experiment::TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, "TEST_ONLY_TCMALLOC_HEAP_PARTITIONING"},

tcmalloc/global_stats.cc

@@ -33,6 +33,7 @@
 #include "tcmalloc/experiment_config.h"
 #include "tcmalloc/guarded_page_allocator.h"
 #include "tcmalloc/huge_page_filler.h"
+#include "tcmalloc/huge_page_options.h"
 #include "tcmalloc/huge_pages.h"
 #include "tcmalloc/internal/config.h"
 #include "tcmalloc/internal/cpu_utils.h"
@@ -636,6 +637,11 @@ void DumpStats(Printer& out, int level) {
                Parameters::release_pages_from_huge_region() ? 1 : 0);
     out.printf("PARAMETER tcmalloc_huge_region_adaptive_release %d\n",
                Parameters::huge_region_adaptive_release() ? 1 : 0);
+    out.printf("PARAMETER madvise_cold_regions_nohugepage %d\n",
+               Parameters::madvise_cold_regions_nohugepage() ==
+                       MadviseRegionsNoHugepage::kEnabled
+                   ? 1
+                   : 0);
     out.printf("PARAMETER tcmalloc_use_wider_slabs %d\n",
                tc_globals.cpu_cache().UseWiderSlabs() ? 1 : 0);
     out.printf("PARAMETER heap_partitioning %d\n",
@@ -916,6 +922,9 @@ void DumpStatsInPbtxt(Printer& out, int level) {
                    Parameters::release_pages_from_huge_region());
   region.PrintBool("tcmalloc_huge_region_adaptive_release",
                    Parameters::huge_region_adaptive_release());
+  region.PrintBool("madvise_cold_regions_nohugepage",
+                   Parameters::madvise_cold_regions_nohugepage() ==
+                       MadviseRegionsNoHugepage::kEnabled);
   region.PrintI64("profile_sampling_interval",
                   Parameters::profile_sampling_interval());
   region.PrintRaw("percpu_vcpu_type",
@@ -963,7 +972,7 @@ void DumpStatsInPbtxt(Printer& out, int level) {
 }
 
 bool GetNumericProperty(const char* name_data, size_t name_size,
-                        size_t* value) {
+                        size_t* absl_nonnull value) {
   TC_ASSERT(name_data != nullptr || name_size == 0);
   TC_ASSERT_NE(value, nullptr);
   const absl::string_view name(name_data, name_size);

tcmalloc/huge_page_aware_allocator.h

@@ -15,7 +15,9 @@
 #ifndef TCMALLOC_HUGE_PAGE_AWARE_ALLOCATOR_H_
 #define TCMALLOC_HUGE_PAGE_AWARE_ALLOCATOR_H_
 
+#include <errno.h>
 #include <stddef.h>
+#include <sys/mman.h>
 
 #include <cstdint>
 #include <optional>
@@ -94,6 +96,10 @@ class StaticForwarder {
     return Parameters::release_stale_pages();
   }
 
+  static MadviseRegionsNoHugepage madvise_cold_regions_nohugepage() {
+    return Parameters::madvise_cold_regions_nohugepage();
+  }
+
   // Arena state.
   static Arena& arena();
 
@@ -844,6 +850,16 @@ template <class Forwarder>
 inline bool HugePageAwareAllocator<Forwarder>::AddRegion() {
   HugeRange r = alloc_.Get(HugeRegion::size());
   if (!r.valid()) return false;
+
+  if (forwarder_.madvise_cold_regions_nohugepage() ==
+      MadviseRegionsNoHugepage::kEnabled) {
+    bool madvise_failed = false;
+    do {
+      madvise_failed =
+          madvise(r.start_addr(), r.len().in_bytes(), MADV_NOHUGEPAGE) != 0;
+    } while (madvise_failed && errno == EAGAIN);
+  }
+
   HugeRegion* region = region_allocator_.New(r, unback_, set_anon_vma_name_);
   regions_.Contribute(region);
   return true;

tcmalloc/huge_page_aware_allocator_fuzz.cc

@@ -299,6 +299,15 @@ struct SetEnableReleaseStalePages {
   }
 };
 
+struct SetMadvNoHugepageHugeRegions {
+  bool value;
+
+  template <typename Sink>
+  friend void AbslStringify(Sink& sink, const SetMadvNoHugepageHugeRegions& s) {
+    absl::Format(&sink, "SetMadvNoHugepageHugeRegions{.value=%v}", s.value);
+  }
+};
+
 struct Instruction;
 
 template <typename Sink>
@@ -314,7 +323,7 @@ using ParamOp = std::variant<
     SetHpaaSubrelease, SetReleaseSucceeds, SetHugeRegionDemandBasedRelease,
     SetHugeRegionAdaptiveRelease, SetBackAllocations, SetBackSizeThresholdBytes,
     ReentrantSubprogram, SetEnableUnfilteredCollapse, SetReleaseMaxColdPages,
-    SetEnableReleaseStalePages>;
+    SetEnableReleaseStalePages, SetMadvNoHugepageHugeRegions>;
 
 template <typename Sink>
 void AbslStringify(Sink& sink, const ParamOp& p) {
@@ -615,6 +624,12 @@ void FuzzHPAA(FuzzHugePageAwareAllocatorOptions fuzz_options,
                       forwarder.set_release_stale_pages(
                           param_arg.value ? ReleaseStalePages::kEnabled
                                           : ReleaseStalePages::kDisabled);
+                    } else if constexpr (std::is_same_v<
+                                             P, SetMadvNoHugepageHugeRegions>) {
+                      forwarder.set_madvise_cold_regions_nohugepage(
+                          param_arg.value
+                              ? MadviseRegionsNoHugepage::kEnabled
+                              : MadviseRegionsNoHugepage::kDisabled);
                     }
                   },
                   arg.op);

tcmalloc/huge_page_options.h

@@ -37,6 +37,11 @@ enum class EnableUnfilteredCollapse : bool {
   kEnabled = true,
 };
 
+enum class MadviseRegionsNoHugepage : bool {
+  kDisabled = false,
+  kEnabled = true,
+};
+
 }  // namespace tcmalloc::tcmalloc_internal
 GOOGLE_MALLOC_SECTION_END
 

tcmalloc/mock_huge_page_static_forwarder.h

@@ -33,6 +33,7 @@
 #include "tcmalloc/arena.h"
 #include "tcmalloc/common.h"
 #include "tcmalloc/huge_page_filler.h"
+#include "tcmalloc/huge_page_options.h"
 #include "tcmalloc/huge_pages.h"
 #include "tcmalloc/internal/config.h"
 #include "tcmalloc/internal/logging.h"
@@ -100,6 +101,14 @@ class FakeStaticForwarder {
     release_stale_pages_ = value;
   }
 
+  MadviseRegionsNoHugepage madvise_cold_regions_nohugepage() const {
+    return madvise_cold_regions_nohugepage_;
+  }
+
+  void set_madvise_cold_regions_nohugepage(MadviseRegionsNoHugepage value) {
+    madvise_cold_regions_nohugepage_ = value;
+  }
+
   bool BackAllocations() const { return back_allocations_; }
   void SetBackAllocations(bool value) { back_allocations_ = value; }
   int32_t BackSizeThresholdBytes() const { return back_size_threshold_bytes_; }
@@ -244,6 +253,8 @@ class FakeStaticForwarder {
       EnableUnfilteredCollapse::kDisabled;
   Arena arena_;
   ReleaseStalePages release_stale_pages_ = ReleaseStalePages::kDisabled;
+  MadviseRegionsNoHugepage madvise_cold_regions_nohugepage_ =
+      MadviseRegionsNoHugepage::kDisabled;
 
   std::atomic<uintptr_t> fake_allocation_ = 0x1000;
 

tcmalloc/parameters.cc

@@ -21,7 +21,6 @@
 
 #include "absl/base/attributes.h"
 #include "absl/base/call_once.h"
-#include "absl/base/const_init.h"
 #include "absl/base/internal/spinlock.h"
 #include "absl/time/time.h"
 #include "tcmalloc/central_freelist.h"
@@ -240,6 +239,20 @@ ABSL_CONST_INIT std::atomic<int32_t> Parameters::back_size_threshold_bytes_(
 ABSL_CONST_INIT std::atomic<bool> Parameters::enable_unfiltered_collapse_(
     false);
 ABSL_CONST_INIT std::atomic<bool> Parameters::release_max_cold_pages_(false);
+static std::atomic<MadviseRegionsNoHugepage>&
+madvise_cold_regions_nohugepage_enabled() {
+  ABSL_CONST_INIT static absl::once_flag flag;
+  ABSL_CONST_INIT static std::atomic<MadviseRegionsNoHugepage> v{
+      MadviseRegionsNoHugepage::kDisabled};
+  absl::base_internal::LowLevelCallOnce(&flag, [&]() {
+    if (IsExperimentActive(
+            Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS)) {
+      v.store(MadviseRegionsNoHugepage::kEnabled, std::memory_order_relaxed);
+    }
+  });
+  return v;
+}
+
 static std::atomic<bool>& huge_region_adaptive_release_enabled() {
   ABSL_CONST_INIT static absl::once_flag flag;
   ABSL_CONST_INIT static std::atomic<bool> v{false};
@@ -321,6 +334,11 @@ bool Parameters::huge_region_adaptive_release() {
   return huge_region_adaptive_release_enabled().load(std::memory_order_relaxed);
 }
 
+MadviseRegionsNoHugepage Parameters::madvise_cold_regions_nohugepage() {
+  return madvise_cold_regions_nohugepage_enabled().load(
+      std::memory_order_relaxed);
+}
+
 HeapPartitioningMode Parameters::heap_partitioning_mode() {
   return heap_partitioning_mode_ptr().load(std::memory_order_relaxed);
 }
@@ -674,6 +692,18 @@ void TCMalloc_Internal_SetReleaseMaxColdPages(bool v) {
   Parameters::release_max_cold_pages_.store(v, std::memory_order_relaxed);
 }
 
+bool TCMalloc_Internal_GetMadviseColdRegionsNoHugepage() {
+  return Parameters::madvise_cold_regions_nohugepage() ==
+         tcmalloc::tcmalloc_internal::MadviseRegionsNoHugepage::kEnabled;
+}
+
+void TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(bool v) {
+  tcmalloc::tcmalloc_internal::madvise_cold_regions_nohugepage_enabled().store(
+      v ? tcmalloc::tcmalloc_internal::MadviseRegionsNoHugepage::kEnabled
+        : tcmalloc::tcmalloc_internal::MadviseRegionsNoHugepage::kDisabled,
+      std::memory_order_relaxed);
+}
+
 }  // extern "C"
 
 GOOGLE_MALLOC_SECTION_END

tcmalloc/parameters.h

@@ -24,6 +24,7 @@
 #include "absl/time/time.h"
 #include "tcmalloc/central_freelist.h"
 #include "tcmalloc/huge_page_filler.h"
+#include "tcmalloc/huge_page_options.h"
 #include "tcmalloc/internal/config.h"
 #include "tcmalloc/internal/logging.h"
 #include "tcmalloc/internal/parameter_accessors.h"
@@ -139,6 +140,12 @@ class Parameters {
     TCMalloc_Internal_SetReleaseMaxColdPages(value);
   }
 
+  static MadviseRegionsNoHugepage madvise_cold_regions_nohugepage();
+
+  static void set_madvise_cold_regions_nohugepage(bool value) {
+    TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(value);
+  }
+
   static void set_per_cpu_caches(bool value) {
 #if !defined(TCMALLOC_DEPRECATED_PERTHREAD)
     if (!value) {

tcmalloc/testing/get_stats_test.cc

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <ctype.h>
 #include <pthread.h>
 #include <stddef.h>
 #include <string.h>
@@ -21,7 +20,6 @@
 #include <cstdint>
 #include <cstdlib>
 #include <limits>
-#include <memory>
 #include <optional>
 #include <set>
 #include <string>
@@ -33,12 +31,9 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/string_view.h"
 #include "absl/time/time.h"
-#include "absl/types/optional.h"
-#include "tcmalloc/common.h"
 #include "tcmalloc/experiment.h"
 #include "tcmalloc/experiment_config.h"
 #include "tcmalloc/global_stats.h"
-#include "tcmalloc/internal/config.h"
 #include "tcmalloc/internal/logging.h"
 #include "tcmalloc/internal/memory_stats.h"
 #include "tcmalloc/malloc_extension.h"
@@ -158,6 +153,12 @@ TEST_F(GetStatsTest, Pbtxt) {
     EXPECT_THAT(buf, HasSubstr("min_hot_access_hint: 1"));
   }
 
+  if (IsExperimentActive(Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS)) {
+    EXPECT_THAT(buf, HasSubstr("madvise_cold_regions_nohugepage: true"));
+  } else {
+    EXPECT_THAT(buf, HasSubstr("madvise_cold_regions_nohugepage: false"));
+  }
+
   EXPECT_THAT(buf, HasSubstr("tcmalloc_enable_unfiltered_collapse: false"));
   if (MallocExtension::PerCpuCachesActive()) {
     EXPECT_THAT(buf, ContainsRegex("cpu_caches_touched: [0-9]+"));
@@ -267,6 +268,14 @@ TEST_F(GetStatsTest, Parameters) {
           buf,
           HasSubstr(R"(PARAMETER tcmalloc_huge_region_adaptive_release 0)"));
     }
+    if (IsExperimentActive(
+            Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS)) {
+      EXPECT_THAT(buf,
+                  HasSubstr(R"(PARAMETER madvise_cold_regions_nohugepage 1)"));
+    } else {
+      EXPECT_THAT(buf,
+                  HasSubstr(R"(PARAMETER madvise_cold_regions_nohugepage 0)"));
+    }
     if (using_hpaa(buf)) {
       EXPECT_THAT(buf, HasSubstr(R"(using_hpaa_subrelease: false)"));
     }

tcmalloc/variants.bzl

@@ -289,6 +289,12 @@ test_variants = [
         "deps": ["//tcmalloc:common_8k_pages"],
         "env": {"BORG_EXPERIMENTS": "TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES"},
     },
+    {
+        "name": "tcmalloc_madv_nohugepage_regions",
+        "malloc": "//tcmalloc",
+        "deps": ["//tcmalloc:common_8k_pages"],
+        "env": {"BORG_EXPERIMENTS": "TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS"},
+    },
 ]
 
 def create_tcmalloc_library(