refactor(tests): Make exec tests connector-agnostic

.github/workflows/pre-commit-checks.yml

@@ -60,6 +60,18 @@ jobs:
       - name: Generate Compile Database
         run: |
           make compile_db_all
+      - name: Ensure generated headers exist
+        run: |
+          # _compile_db (invoked by compile_db_all) is supposed to build these,
+          # but make it explicit so clang-tidy in the next step never races
+          # against a missing dwrf_schema.pb.h / parquet thrift header. If the
+          # underlying chain already produced them, this is a no-op rebuild.
+          cmake --build --preset conan-release \
+            --target generate_parquet_thrift \
+            --target bolt_dwio_dwrf_proto
+          test -f _build/Release/bolt/dwio/dwrf/proto/dwrf_schema.pb.h \
+            || { echo "::error::dwrf_schema.pb.h missing after proto build" >&2; \
+                 ls -la _build/Release/bolt/dwio/dwrf/proto/ || true; exit 1; }
       - name: Run pre-commit hooks
         run: |
           pre-commit run --all-files

bolt/benchmarks/QueryBenchmarkBase.cpp

@@ -30,6 +30,8 @@
 
 #include "bolt/benchmarks/QueryBenchmarkBase.h"
 
+#include "bolt/common/caching/SsdCache.h"
+
 DEFINE_string(data_format, "parquet", "Data format");
 
 DEFINE_validator(

bolt/common/base/tests/FsTest.cpp

@@ -30,14 +30,15 @@
 
 #include "bolt/common/base/Fs.h"
 #include <gtest/gtest.h>
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
 #include "boost/filesystem.hpp"
 namespace bytedance::bolt::common {
 
 class FsTest : public testing::Test {};
 
 TEST_F(FsTest, createDirectory) {
-  auto rootPath = exec::test::TempDirectoryPath::createTempDirectory();
+  auto rootPath =
+      bytedance::bolt::test::TempDirectoryPath::createTempDirectory();
   auto tmpDirectoryPath = rootPath + "/first/second/third";
   // First time should generate directory successfully.
   EXPECT_FALSE(fs::exists(tmpDirectoryPath.c_str()));

bolt/common/caching/tests/AsyncDataCacheTest.cpp

@@ -39,8 +39,8 @@
 #ifndef NDEBUG
 #include "bolt/common/testutil/ScopedTestTime.h"
 #endif
+#include "bolt/common/testutil/TempDirectoryPath.h"
 #include "bolt/common/testutil/TestValue.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
 #include "folly/experimental/EventCount.h"
 
 #include <folly/executors/IOThreadPoolExecutor.h>
@@ -118,7 +118,7 @@ class AsyncDataCacheTest : public testing::Test {
       // second creation of cache must find the checkpoint of the
       // previous one.
       if (tempDirectory_ == nullptr) {
-        tempDirectory_ = exec::test::TempDirectoryPath::create();
+        tempDirectory_ = bytedance::bolt::test::TempDirectoryPath::create();
       }
       ssdCache = std::make_unique<SsdCache>(
           fmt::format("{}/cache", tempDirectory_->path),
@@ -258,7 +258,7 @@ class AsyncDataCacheTest : public testing::Test {
     }
   }
 
-  std::shared_ptr<exec::test::TempDirectoryPath> tempDirectory_;
+  std::shared_ptr<bytedance::bolt::test::TempDirectoryPath> tempDirectory_;
   std::unique_ptr<memory::MemoryManager> manager_;
   memory::MemoryAllocator* allocator_;
   std::shared_ptr<AsyncDataCache> cache_;

bolt/common/caching/tests/SsdFileTest.cpp

@@ -31,7 +31,7 @@
 #include "bolt/common/caching/FileIds.h"
 #include "bolt/common/caching/SsdCache.h"
 #include "bolt/common/memory/Memory.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
 
 #include <folly/executors/QueuedImmediateExecutor.h>
 #include <glog/logging.h>
@@ -79,7 +79,7 @@ class SsdFileTest : public testing::Test {
 
     fileName_ = StringIdLease(fileIds(), "fileInStorage");
 
-    tempDirectory_ = exec::test::TempDirectoryPath::create();
+    tempDirectory_ = bytedance::bolt::test::TempDirectoryPath::create();
     ssdFile_ = std::make_unique<SsdFile>(
         fmt::format("{}/ssdtest", tempDirectory_->path),
         0, // shardId
@@ -243,7 +243,7 @@ class SsdFileTest : public testing::Test {
     readAndCheckPins(pins);
   }
 
-  std::shared_ptr<exec::test::TempDirectoryPath> tempDirectory_;
+  std::shared_ptr<bytedance::bolt::test::TempDirectoryPath> tempDirectory_;
 
   std::shared_ptr<AsyncDataCache> cache_;
   StringIdLease fileName_;

bolt/common/file/tests/AsyncFileTest.cpp

@@ -18,8 +18,8 @@
 
 #include "bolt/common/file/File.h"
 #include "bolt/common/file/FileSystems.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
-#include "bolt/exec/tests/utils/TempFilePath.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
+#include "bolt/common/testutil/TempFilePath.h"
 
 #include "gtest/gtest.h"
 using namespace bytedance::bolt;
@@ -96,7 +96,7 @@ void readDataAsyncForWirteBuffers(
 }
 
 TEST(AsyncLocalFile, writeAndRead) {
-  auto tempFile = ::exec::test::TempFilePath::create();
+  auto tempFile = ::bytedance::bolt::test::TempFilePath::create();
   const auto& filename = tempFile->path.c_str();
   remove(filename);
   {
@@ -109,7 +109,7 @@ TEST(AsyncLocalFile, writeAndRead) {
 
 TEST(AsyncLocalFile, viaRegistry) {
   filesystems::registerLocalFileSystem();
-  auto tempFile = ::exec::test::TempFilePath::create();
+  auto tempFile = ::bytedance::bolt::test::TempFilePath::create();
   const auto& filename = tempFile->path.c_str();
   remove(filename);
   auto lfs = filesystems::getFileSystem(filename, nullptr);
@@ -124,7 +124,7 @@ TEST(AsyncLocalFile, viaRegistry) {
 
 TEST(AsyncLocalFileWrite, viaWriteBuffer) {
   filesystems::registerLocalFileSystem();
-  auto tempFile = ::exec::test::TempFilePath::create();
+  auto tempFile = ::bytedance::bolt::test::TempFilePath::create();
   const auto& filename = tempFile->path.c_str();
   remove(filename);
   auto lfs = filesystems::getFileSystem(filename, nullptr);

bolt/common/file/tests/FileInputStreamTest.cpp

@@ -34,7 +34,7 @@
 #include "bolt/common/file/FileInputStream.h"
 #include "bolt/common/file/FileSystems.h"
 #include "bolt/common/memory/MmapAllocator.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
 
 #include <vector>
 
@@ -59,7 +59,7 @@ class FileInputStreamTest : public testing::Test {
     mmapAllocator_ = static_cast<MmapAllocator*>(memoryManager_->allocator());
     pool_ = memoryManager_->addLeafPool("ByteStreamTest");
     rng_.seed(124);
-    tempDirPath_ = exec::test::TempDirectoryPath::create();
+    tempDirPath_ = bytedance::bolt::test::TempDirectoryPath::create();
     fs_ = filesystems::getFileSystem(tempDirPath_->getPath(), nullptr);
   }
 
@@ -87,7 +87,7 @@ class FileInputStreamTest : public testing::Test {
   MmapAllocator* mmapAllocator_;
   std::shared_ptr<memory::MemoryPool> pool_;
   std::atomic_uint64_t fileId_{0};
-  std::shared_ptr<exec::test::TempDirectoryPath> tempDirPath_;
+  std::shared_ptr<bytedance::bolt::test::TempDirectoryPath> tempDirPath_;
   std::shared_ptr<filesystems::FileSystem> fs_;
 };
 

bolt/common/file/tests/FileTest.cpp

@@ -36,8 +36,8 @@
 #include "bolt/common/config/Config.h"
 #include "bolt/common/file/File.h"
 #include "bolt/common/file/FileSystems.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
-#include "bolt/exec/tests/utils/TempFilePath.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
+#include "bolt/common/testutil/TempFilePath.h"
 
 #include "gtest/gtest.h"
 using namespace bytedance::bolt;
@@ -190,7 +190,7 @@ TEST(InMemoryFile, preadv) {
 
 TEST(LocalFile, writeAndRead) {
   for (bool useIOBuf : {true, false}) {
-    auto tempFile = ::exec::test::TempFilePath::create();
+    auto tempFile = ::bytedance::bolt::test::TempFilePath::create();
     const auto& filename = tempFile->path.c_str();
     remove(filename);
     {
@@ -220,7 +220,7 @@ TEST(LocalFile, writeAndRead) {
 
 TEST(LocalFile, viaRegistry) {
   filesystems::registerLocalFileSystem();
-  auto tempFile = ::exec::test::TempFilePath::create();
+  auto tempFile = ::bytedance::bolt::test::TempFilePath::create();
   const auto& filename = tempFile->path.c_str();
   remove(filename);
   auto lfs = filesystems::getFileSystem(filename, nullptr);
@@ -237,7 +237,7 @@ TEST(LocalFile, viaRegistry) {
 
 TEST(LocalFile, rename) {
   filesystems::registerLocalFileSystem();
-  auto tempFolder = ::exec::test::TempDirectoryPath::create();
+  auto tempFolder = ::test::TempDirectoryPath::create();
   auto a = fmt::format("{}/a", tempFolder->path);
   auto b = fmt::format("{}/b", tempFolder->path);
   auto newA = fmt::format("{}/newA", tempFolder->path);
@@ -265,7 +265,7 @@ TEST(LocalFile, rename) {
 
 TEST(LocalFile, exists) {
   filesystems::registerLocalFileSystem();
-  auto tempFolder = ::exec::test::TempDirectoryPath::create();
+  auto tempFolder = ::test::TempDirectoryPath::create();
   auto a = fmt::format("{}/a", tempFolder->path);
   auto b = fmt::format("{}/b", tempFolder->path);
   auto localFs = filesystems::getFileSystem(a, nullptr);
@@ -285,7 +285,7 @@ TEST(LocalFile, exists) {
 
 TEST(LocalFile, list) {
   filesystems::registerLocalFileSystem();
-  auto tempFolder = ::exec::test::TempDirectoryPath::create();
+  auto tempFolder = ::test::TempDirectoryPath::create();
   auto a = fmt::format("{}/1", tempFolder->path);
   auto b = fmt::format("{}/2", tempFolder->path);
   auto localFs = filesystems::getFileSystem(a, nullptr);
@@ -305,7 +305,7 @@ TEST(LocalFile, list) {
 }
 
 TEST(LocalFile, readFileDestructor) {
-  auto tempFile = ::exec::test::TempFilePath::create();
+  auto tempFile = ::bytedance::bolt::test::TempFilePath::create();
   const auto& filename = tempFile->path.c_str();
   remove(filename);
   {
@@ -338,7 +338,7 @@ TEST(LocalFile, readFileDestructor) {
 
 TEST(LocalFile, mkdir) {
   filesystems::registerLocalFileSystem();
-  auto tempFolder = ::exec::test::TempDirectoryPath::create();
+  auto tempFolder = ::test::TempDirectoryPath::create();
 
   std::string path = tempFolder->path;
   auto localFs = filesystems::getFileSystem(path, nullptr);
@@ -364,7 +364,7 @@ TEST(LocalFile, mkdir) {
 
 TEST(LocalFile, rmdir) {
   filesystems::registerLocalFileSystem();
-  auto tempFolder = ::exec::test::TempDirectoryPath::create();
+  auto tempFolder = ::test::TempDirectoryPath::create();
 
   std::string path = tempFolder->path;
   auto localFs = filesystems::getFileSystem(path, nullptr);
@@ -398,7 +398,7 @@ TEST(LocalFile, rmdir) {
 
 TEST(LocalFile, fileNotFound) {
   filesystems::registerLocalFileSystem();
-  auto tempFolder = ::exec::test::TempDirectoryPath::create();
+  auto tempFolder = ::test::TempDirectoryPath::create();
   auto path = fmt::format("{}/file", tempFolder->path);
   auto localFs = filesystems::getFileSystem(path, nullptr);
   BOLT_ASSERT_RUNTIME_THROW_CODE(

bolt/common/memory/tests/ArbitrationParticipantTest.cpp

@@ -46,10 +46,10 @@
 #include "bolt/common/memory/Memory.h"
 #include "bolt/common/memory/MemoryArbitrator.h"
 #include "bolt/common/memory/SharedArbitrator.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
 #include "bolt/common/testutil/TestValue.h"
 #include "bolt/exec/OperatorUtils.h"
 #include "bolt/exec/tests/utils/PlanBuilder.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
 #include "gmock/gmock-matchers.h"
 
 DECLARE_bool(bolt_memory_leak_check_enabled);
@@ -59,6 +59,7 @@ using namespace ::testing;
 using namespace bytedance::bolt::common::testutil;
 using namespace bytedance::bolt::exec;
 using namespace bytedance::bolt::exec::test;
+using namespace bytedance::bolt::test;
 namespace bytedance::bolt::memory {
 static const std::string arbitratorKind("TEST");
 

bolt/common/memory/tests/ByteStreamTest.cpp

@@ -34,7 +34,7 @@
 #include "bolt/common/file/FileInputStream.h"
 #include "bolt/common/file/FileSystems.h"
 #include "bolt/common/memory/MmapAllocator.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
 
 #include <gflags/gflags.h>
 #include <gtest/gtest.h>
@@ -342,7 +342,7 @@ class InputByteStreamTest : public ByteStreamTest,
 
   void SetUp() override {
     ByteStreamTest::SetUp();
-    tempDirPath_ = exec::test::TempDirectoryPath::create();
+    tempDirPath_ = bytedance::bolt::test::TempDirectoryPath::create();
     fs_ = filesystems::getFileSystem(tempDirPath_->getPath(), nullptr);
   }
 
@@ -366,7 +366,7 @@ class InputByteStreamTest : public ByteStreamTest,
   }
 
   std::atomic_uint64_t fileId_{0};
-  std::shared_ptr<exec::test::TempDirectoryPath> tempDirPath_;
+  std::shared_ptr<bytedance::bolt::test::TempDirectoryPath> tempDirPath_;
   std::shared_ptr<filesystems::FileSystem> fs_;
 };
 

bolt/common/memory/tests/MockSharedArbitratorTest.cpp

@@ -40,10 +40,10 @@
 #include "bolt/common/memory/MemoryArbitrator.h"
 #include "bolt/common/memory/SharedArbitrator.h"
 #include "bolt/common/memory/tests/SharedArbitratorTestUtil.h"
+#include "bolt/common/testutil/TempDirectoryPath.h"
 #include "bolt/common/testutil/TestValue.h"
 #include "bolt/exec/OperatorUtils.h"
 #include "bolt/exec/tests/utils/PlanBuilder.h"
-#include "bolt/exec/tests/utils/TempDirectoryPath.h"
 #include "folly/experimental/EventCount.h"
 
 DECLARE_bool(bolt_memory_leak_check_enabled);
@@ -53,6 +53,7 @@ using namespace ::testing;
 using namespace bytedance::bolt::common::testutil;
 using namespace bytedance::bolt::exec;
 using namespace bytedance::bolt::exec::test;
+using namespace bytedance::bolt::test;
 namespace bytedance::bolt::memory {
 // Class to write runtime stats in the tests to the stats container.
 class TestRuntimeStatWriter : public BaseRuntimeStatWriter {

bolt/common/memory/tests/SharedArbitratorTest.cpp

@@ -60,6 +60,7 @@ using namespace ::testing;
 using namespace bytedance::bolt::common::testutil;
 using namespace bytedance::bolt::exec;
 using namespace bytedance::bolt::exec::test;
+using namespace bytedance::bolt::test;
 namespace bytedance::bolt::memory {
 // Custom node for the custom factory.
 class FakeMemoryNode : public core::PlanNode {
@@ -372,7 +373,8 @@ DEBUG_ONLY_TEST_P(
         queryCtxStateChecked = true;
       })));
 
-  const auto spillDirectory = exec::test::TempDirectoryPath::create();
+  const auto spillDirectory =
+      bytedance::bolt::test::TempDirectoryPath::create();
   TestScopedSpillInjection scopedSpillInjection(100);
   core::PlanNodeId aggregationNodeId;
   newQueryBuilder()
@@ -420,7 +422,8 @@ DEBUG_ONLY_TEST_P(
       })));
 
   std::thread queryThread([&] {
-    const auto spillDirectory = exec::test::TempDirectoryPath::create();
+    const auto spillDirectory =
+        bytedance::bolt::test::TempDirectoryPath::create();
     core::PlanNodeId aggregationNodeId;
     auto plan = PlanBuilder()
                     .values(vectors)
@@ -499,7 +502,8 @@ DEBUG_ONLY_TEST_P(
                              .singleAggregation({"c0", "c1"}, {"array_agg(c2)"})
                              .planNode();
   std::thread spillableThread([&]() {
-    const auto spillDirectory = exec::test::TempDirectoryPath::create();
+    const auto spillDirectory =
+        bytedance::bolt::test::TempDirectoryPath::create();
     newQueryBuilder(spillPlan)
         .queryCtx(queryCtx)
         .spillDirectory(spillDirectory->getPath())
@@ -954,7 +958,8 @@ DEBUG_ONLY_TEST_P(
       })));
 
   const int numDrivers = 1;
-  const auto spillDirectory = exec::test::TempDirectoryPath::create();
+  const auto spillDirectory =
+      bytedance::bolt::test::TempDirectoryPath::create();
   std::thread queryThread([&]() {
     BOLT_ASSERT_THROW(
         newQueryBuilder()
@@ -1027,7 +1032,8 @@ DEBUG_ONLY_TEST_P(
             [&]() { return aggregationAllocationUnblocked.load(); });
       })));
 
-  const auto spillDirectory = exec::test::TempDirectoryPath::create();
+  const auto spillDirectory =
+      bytedance::bolt::test::TempDirectoryPath::create();
   std::shared_ptr<Task> task;
   std::thread queryThread([&]() {
     task = newQueryBuilder()
@@ -1097,7 +1103,8 @@ DEBUG_ONLY_TEST_P(SharedArbitrationTestWithThreadingModes, runtimeStats) {
             values->pool()->free(buffer, fakeAllocationSize);
           })));
 
-  const auto spillDirectory = exec::test::TempDirectoryPath::create();
+  const auto spillDirectory =
+      bytedance::bolt::test::TempDirectoryPath::create();
   const auto outputDirectory = TempDirectoryPath::create();
   const auto queryCtx =
       newQueryCtx(memoryManager_.get(), executor_.get(), memoryCapacity);