CrunchyData · aykut-bozkurt · Oct 27, 2025 · Oct 31, 2025 · Nov 2, 2025
diff --git a/.devcontainer/.env b/.devcontainer/.env
@@ -2,7 +2,7 @@
 AWS_ACCESS_KEY_ID=minioadmin
 AWS_SECRET_ACCESS_KEY=minioadmin
 AWS_REGION=us-east-1
-AWS_ENDPOINT_URL=http://localhost:9000
+AWS_ENDPOINT_URL=http://minio:9000
 AWS_ENDPOINT_PROXY_PORT=9999
 AWS_ALLOW_HTTP=true
 AWS_S3_TEST_BUCKET=testbucket
@@ -12,24 +12,44 @@ MINIO_ROOT_PASSWORD=minioadmin
 # Azure Blob tests
 AZURE_STORAGE_ACCOUNT=devstoreaccount1
 AZURE_STORAGE_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
-AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"
-AZURE_STORAGE_ENDPOINT=http://localhost:10000/devstoreaccount1
+AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;"
+AZURE_STORAGE_ENDPOINT=http://azurite:10000/devstoreaccount1
 AZURE_ALLOW_HTTP=true
 AZURE_TEST_CONTAINER_NAME=testcontainer
 AZURE_TEST_READ_ONLY_SAS=se=2100-05-05&sp=r&sv=2022-11-02&sr=c&sig=YMPFnAHKe9y0o3hFegncbwQTXtAyvsJEgPB2Ne1b9CQ%3D
 AZURE_TEST_READ_WRITE_SAS=se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t9L651t6rTCQr%2BOjmJHkM76tnCGdcyttnlA%3D
 
 # http(s) tests
 ALLOW_HTTP=true
-HTTP_ENDPOINT=http://localhost:8080
+HTTP_ENDPOINT=http://webdav:8080
 
 # GCS tests
 GOOGLE_TEST_BUCKET=testbucket
-GOOGLE_SERVICE_ACCOUNT_KEY={"gcs_base_url":"http://localhost:4443","disable_oauth":true,"client_email":"","private_key_id":"","private_key":""}
-GOOGLE_SERVICE_ENDPOINT=http://localhost:4443
+GOOGLE_SERVICE_ACCOUNT_KEY={"gcs_base_url":"http://fake-gcs-server:4443","disable_oauth":true,"client_email":"","private_key_id":"","private_key":""}
+GOOGLE_SERVICE_ENDPOINT=http://fake-gcs-server:4443
 
 # Others
+
 # run pgrx tests with a single thread to avoid race conditions
 RUST_TEST_THREADS=1
+
 # pgrx runs test on the port base_port + pg_version
 PGRX_TEST_PG_BASE_PORT=5454
+
+# default PostgreSQL version
+DEFAULT_PG_MAJOR=18
+
+# Rust version
+RUSTC_VERSION=1.90.0
+
+# rust build flags
+RUST_BACKTRACE=1
+CARGO_INCREMENTAL=0
+
+# Docker images
+MINIO_IMAGE=docker.io/minio/minio
+AZURITE_IMAGE=mcr.microsoft.com/azure-storage/azurite
+FAKE_GCS_IMAGE=docker.io/tustvold/fake-gcs-server
+WEBDAV_IMAGE=docker.io/rclone/rclone
+BASE_IMAGE_OS=almalinux
+BASE_IMAGE_OS_VERSION=9
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,64 +1,170 @@
-FROM ubuntu:22.04
+ARG BASE_IMAGE_OS=almalinux
+ARG BASE_IMAGE_OS_VERSION=9
+FROM ${BASE_IMAGE_OS}:${BASE_IMAGE_OS_VERSION}
+ARG BASE_IMAGE_OS
+ARG BASE_IMAGE_OS_VERSION
 
 ENV DEBIAN_FRONTEND="noninteractive"
 ENV TZ="Europe/Istanbul"
 
-ARG PG_MAJOR=18
+ARG DEFAULT_PG_MAJOR=18
 
-# install deps
-RUN apt-get update && apt-get -y install build-essential libreadline-dev zlib1g-dev      \
-                                         flex bison libxml2-dev libxslt-dev libssl-dev   \
-                                         libxml2-utils xsltproc ccache pkg-config wget   \
-                                         curl lsb-release ca-certificates gnupg sudo git \
-                                         nano net-tools awscli libkrb5-dev
+SHELL ["/bin/bash", "-c"]
 
-# install azure-cli
-RUN curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null
-RUN echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ `lsb_release -cs` main" | tee /etc/apt/sources.list.d/azure-cli.list
-RUN apt-get update && apt-get install -y azure-cli
+# install deps
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+      apt-get update && \
+      apt-get -y install build-essential libreadline-dev zlib1g-dev flex bison         \
+                         libxml2-dev libxslt-dev libssl-dev git libxml2-utils xsltproc \
+                         pkg-config wget curl sudo lsb-release ca-certificates gnupg   \
+                         nano net-tools libkrb5-dev python3 python3-pip python3-venv   \
+                         python3-dev &&                                                \
+      apt install --only-upgrade python3-pip && \
+      pip3 install pipx --break-system-packages; \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+      dnf -y update && \
+      dnf -y install epel-release && \
+      (dnf config-manager --set-enabled powertools || true) && \
+      (dnf config-manager --set-enabled crb || true) && \
+      dnf -y install gcc gcc-c++ make readline-devel zlib-devel flex bison libxml2-devel    \
+                     libxslt-devel openssl-devel pkgconfig wget ca-certificates sudo git    \
+                     which gnupg nano net-tools krb5-devel python3 python3-pip              \
+                     platform-python-devel &&                                               \
+      pip3 install --upgrade pip && \
+      pip3 install pipx; \
+    fi
 
 # install Postgres
-RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN apt-get update && apt-get -y install postgresql-${PG_MAJOR}-postgis-3  \
-                                         postgresql-server-dev-${PG_MAJOR} \
-                                         postgresql-client-${PG_MAJOR}     \
-                                         libpq-dev
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+      wget -O- https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > postgresql-keyring.gpg && \
+      cat postgresql-keyring.gpg | sudo tee -a /usr/share/keyrings/postgresql-keyring.gpg > /dev/null && \
+      sh -c 'echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
+      apt-get update && apt-get -y install postgresql-{14,15,16,17,18}-postgis-3  \
+                                           postgresql-{14,15,16,17,18}-pgaudit    \
+                                           postgresql-server-dev-{14,15,16,17,18} \
+                                           postgresql-client-{14,15,16,17,18}     \
+                                           libpq-dev;                             \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+      ARCH=$(uname -m) && \
+      dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-$(rpm -E %{rhel})-${ARCH}/pgdg-redhat-repo-latest.noarch.rpm && \
+      (dnf -qy module disable postgresql || true) && \
+      dnf install -y --nogpgcheck \
+                     postgis36_{14,15,16,17,18}        \
+                     pgaudit{16_14,17_15,_16,_17,_18}  \
+                     postgresql{14,15,16,17,18}-server \
+                     postgresql{14,15,16,17,18}-libs   \
+                     postgresql{14,15,16,17,18}-devel; \
+    fi
+
+# default pg version for debian
+ENV PATH="/usr/lib/postgresql/${DEFAULT_PG_MAJOR}/bin:${PATH}"
+
+# default pg version for almalinux
+ENV PATH="/usr/pgsql-${DEFAULT_PG_MAJOR}/bin/:${PATH}"
 
 # set up permissions so that rust user can create extensions
-RUN chmod a+rwx `pg_config --pkglibdir`          \
-                `pg_config --pkglibdir`/bitcode  \
-                `pg_config --sharedir`/extension \
-                /var/run/postgresql/
-
-# install pgaudit extension
-RUN if [ ${PG_MAJOR} != "18" ]; then \
-      apt-get -y install postgresql-${PG_MAJOR}-pgaudit; \
-    else \
-      git clone --branch integration https://github.com/pgaudit/pgaudit.git /tmp/pgaudit && \
-      cd /tmp/pgaudit && make install USE_PGXS=1 PG_CONFIG=pg_config; \
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+        chmod a+rwx `/usr/lib/postgresql/14/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/14/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/14/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/15/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/15/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/15/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/16/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/16/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/16/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/17/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/17/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/17/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/18/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/18/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/18/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/; \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+        chmod a+rwx `/usr/pgsql-14/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-14/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-14/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-15/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-15/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-15/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-16/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-16/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-16/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-17/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-17/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-17/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-18/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-18/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-18/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/; \
     fi
 
 # initdb requires non-root user. This will also be the user that runs the container.
 ARG USERNAME=rust
-ARG USER_UID=1000
-ARG USER_GID=1000
+ARG USER_UID=1001
+ARG USER_GID=1001
 RUN groupadd --gid $USER_GID $USERNAME
 RUN useradd --uid $USER_UID --gid $USER_GID -s /bin/bash -m $USERNAME
 
 RUN echo "$USERNAME ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USERNAME
 
 USER $USERNAME
 
+# Install awscli, azure-cli and mitmdump with rust user
+RUN pipx install mitmproxy && pipx install awscli && pipx install azure-cli
+ENV PATH="~/.local/bin:${PATH}"
+
 # install Rust environment
+ARG RUSTC_VERSION=1.90.0
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y
+
 ENV PATH="/home/rust/.cargo/bin:${PATH}"
 
+RUN rustup install ${RUSTC_VERSION} && \
+    rustup default ${RUSTC_VERSION} && \
+    rustup component add rustfmt clippy llvm-tools-preview
+
+# install llvm-cov for coverage reports
+RUN cargo install --locked cargo-llvm-cov@0.6.12
+
 # install and configure pgrx
 ARG PGRX_VERSION=0.16.1
 RUN cargo install --locked cargo-pgrx@${PGRX_VERSION}
-RUN cargo pgrx init --pg${PG_MAJOR} $(which pg_config)
-RUN echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-${PG_MAJOR}/postgresql.conf
+
+ENV PGRX_TEST_PG_BASE_PORT=5454
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+        cargo pgrx init --pg14 /usr/lib/postgresql/14/bin/pg_config \
+                        --pg15 /usr/lib/postgresql/15/bin/pg_config \
+                        --pg16 /usr/lib/postgresql/16/bin/pg_config \
+                        --pg17 /usr/lib/postgresql/17/bin/pg_config \
+                        --pg18 /usr/lib/postgresql/18/bin/pg_config \
+                        --base-testing-port ${PGRX_TEST_PG_BASE_PORT}; \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+        cargo pgrx init --pg14 /usr/pgsql-14/bin/pg_config \
+                        --pg15 /usr/pgsql-15/bin/pg_config \
+                        --pg16 /usr/pgsql-16/bin/pg_config \
+                        --pg17 /usr/pgsql-17/bin/pg_config \
+                        --pg18 /usr/pgsql-18/bin/pg_config \
+                        --base-testing-port ${PGRX_TEST_PG_BASE_PORT}; \
+    fi
+
+RUN echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-14/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-15/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-16/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-17/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-18/postgresql.conf
 
 # required for pgrx to work
 ENV USER=$USERNAME

diff --git a/.devcontainer/README.md b/.devcontainer/README.md
@@ -0,0 +1,23 @@
+## Devcontainer
+If you install devcontainer extension at vscode, you can spin up your development environment via pg_parquet devcontainer. All necessary components and dependencies would be installed in your environment after the spin-up. You can work in the container as if it is your development environment. (git, make, aws cli and postgres commands should work seamlessly) The container offers reproducible environment as we use the same dockerfile for CI and devcontainer.
+
+## Prerequisites
+1. Make sure to allocate >= ~10GB memory resource to docker.
+2. Your `~/.aws` folder is mounted as a read only volume to pg_parquet container so that it can read/write buckets with your aws credentials. You are expected to set up your aws credentials before. (possible to read/write from/to **production** s3 buckets)
+
+After opening the project inside the devcontainer, you can do the following steps:
+
+1. Connect to the postgres server (build takes place at the first run)
+```bash
+cargo pgrx run
+```
+2. Create pg_parquet extension
+```sql
+pg_parquet=# create extension pg_parquet;
+CREATE EXTENSION
+```
+3. Copy test data to parquet file
+```sql
+pg_parquet=# copy (select i from generate_series(1, 10) i) to '/tmp/test.parquet';
+COPY 10
+```
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,9 +1,10 @@
 {
     "name": "pg_parquet Dev Environment",
     "dockerComposeFile": "docker-compose.yml",
-    "service": "app",
+    "service": "pg_parquet",
     "workspaceFolder": "/workspace",
-    "postAttachCommand": "sudo chown -R rust /workspace",
+    "remoteUser": "rust",
+    "containerUser": "rust",
     "customizations": {
         "vscode": {
             "extensions": [