diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 356faeebfb0..e558ad5ecdb 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -1,317 +1,317 @@ -name: Build Linux arm64 +# name: Build Linux arm64 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (Linux ARM64) - runs-on: GH-Linux-ARM64 - steps: - - name: Install Python build dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - sudo apt-get update && sudo apt-get install -y jq lsb-release +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (Linux ARM64) +# runs-on: GH-Linux-ARM64 +# steps: +# - name: Install Python build dependencies +# run: | +# sudo apt-get update +# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ +# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ +# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ +# libffi-dev liblzma-dev +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# sudo apt-get update && sudo apt-get install -y jq lsb-release - # Detect OS distribution info - DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') - DISTRO_VERSION=$(lsb_release -sr) - echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" +# # Detect OS distribution info +# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +# DISTRO_VERSION=$(lsb_release -sr) +# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - pyenv shell --unset - done - - name: Install clang++ for Ubuntu - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - which clang++-19 - clang++-19 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ubuntu-24.04-aarch64 - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-19 to clang - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-19 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel +# pyenv shell --unset +# done +# - name: Install clang++ for Ubuntu +# run: | +# pwd +# uname -a +# wget https://apt.llvm.org/llvm.sh +# chmod +x llvm.sh +# sudo ./llvm.sh 19 +# which clang++-19 +# clang++-19 --version +# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget +# ccache -s +# - name: Update git +# run: | +# sudo add-apt-repository ppa:git-core/ppa -y +# sudo apt-get update +# sudo apt-get install -y git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ubuntu-24.04-aarch64 +# max-size: 5G +# append-timestamp: true +# - name: remove old clang and link clang-19 to clang +# run: | +# sudo rm -f /usr/bin/clang || true +# sudo ln -s /usr/bin/clang-19 /usr/bin/clang +# sudo rm -f /usr/bin/clang++ || true +# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - sudo rm -f dist/*linux_aarch64.whl - ls -lh dist - shell: bash - - name: Setup core dump collection - run: | - mkdir -p tmp/core - echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-linux-aarch64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-linux-aarch64.tar.gz" - ls -lh core-files-linux-aarch64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files if present - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-linux-aarch64 - path: core-files-linux-aarch64.tar.gz - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-aarch64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-aarch64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-aarch64 - path: | - ./dist/*.whl - ./linux-aarch64-libchdb.tar.gz - ./linux-aarch64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# sudo cp bin/patchelf /usr/bin/ +# sudo chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# python3 -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +# sudo rm -f dist/*linux_aarch64.whl +# ls -lh dist +# shell: bash +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-linux-aarch64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-linux-aarch64.tar.gz" +# ls -lh core-files-linux-aarch64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files if present +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-linux-aarch64 +# path: core-files-linux-aarch64.tar.gz +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-aarch64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-aarch64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-linux-aarch64 +# path: | +# ./dist/*.whl +# ./linux-aarch64-libchdb.tar.gz +# ./linux-aarch64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 665a8a3f2b8..e6146f4bfba 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -1,316 +1,316 @@ -name: Build Linux X86 +# name: Build Linux X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (Linux x86_64) - runs-on: gh-64c - steps: - - name: Install Python build dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - sudo apt-get update && sudo apt-get install -y jq lsb-release +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (Linux x86_64) +# runs-on: gh-64c +# steps: +# - name: Install Python build dependencies +# run: | +# sudo apt-get update +# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ +# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ +# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ +# libffi-dev liblzma-dev +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# sudo apt-get update && sudo apt-get install -y jq lsb-release - # Detect OS distribution info - DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') - DISTRO_VERSION=$(lsb_release -sr) - echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" +# # Detect OS distribution info +# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +# DISTRO_VERSION=$(lsb_release -sr) +# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - pyenv shell --unset - done - - name: Install clang++ for Ubuntu - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - which clang++-19 - clang++-19 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ubuntu-22.04-x86_64 - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-19 to clang - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-19 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel +# pyenv shell --unset +# done +# - name: Install clang++ for Ubuntu +# run: | +# pwd +# uname -a +# wget https://apt.llvm.org/llvm.sh +# chmod +x llvm.sh +# sudo ./llvm.sh 19 +# which clang++-19 +# clang++-19 --version +# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget +# ccache -s +# - name: Update git +# run: | +# sudo add-apt-repository ppa:git-core/ppa -y +# sudo apt-get update +# sudo apt-get install -y git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ubuntu-22.04-x86_64 +# max-size: 5G +# append-timestamp: true +# - name: remove old clang and link clang-19 to clang +# run: | +# sudo rm -f /usr/bin/clang || true +# sudo ln -s /usr/bin/clang-19 /usr/bin/clang +# sudo rm -f /usr/bin/clang++ || true +# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - sudo rm -f dist/*-linux_x86_64.whl - ls -lh dist - shell: bash - - name: Setup core dump collection - run: | - mkdir -p tmp/core - echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-linux-x86_64.tar.gz" - ls -lh core-files-linux-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-linux-x86_64 - path: core-files-linux-x86_64.tar.gz - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-x86_64 - path: | - ./dist/*.whl - ./linux-x86_64-libchdb.tar.gz - ./linux-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# sudo cp bin/patchelf /usr/bin/ +# sudo chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# python3 -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# sudo rm -f dist/*-linux_x86_64.whl +# ls -lh dist +# shell: bash +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-linux-x86_64.tar.gz" +# ls -lh core-files-linux-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-linux-x86_64 +# path: core-files-linux-x86_64.tar.gz +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-linux-x86_64 +# path: | +# ./dist/*.whl +# ./linux-x86_64-libchdb.tar.gz +# ./linux-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index a7d7fb454e8..0c9bfa70ca8 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -1,319 +1,319 @@ -name: Build macOS arm64 +# name: Build macOS arm64 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS ARM64) - runs-on: macos-13-xlarge - steps: - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS ARM64) +# runs-on: macos-13-xlarge +# steps: +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd - brew install openssl@3 || echo "OpenSSL install failed, continuing..." - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext binutils grep findutils nasm - # brew install gcc || echo "GCC install failed, continuing..." - # brew install ccache || echo "ccache installation failed, continuing without it" - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$PATH - which clang++ - clang++ --version - which go - go version - ccache -s || echo "ccache not available yet" - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd +# brew install openssl@3 || echo "OpenSSL install failed, continuing..." +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext binutils grep findutils nasm +# # brew install gcc || echo "GCC install failed, continuing..." +# # brew install ccache || echo "ccache installation failed, continuing without it" +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$PATH +# which clang++ +# clang++ --version +# which go +# go version +# ccache -s || echo "ccache not available yet" +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in Homebrew packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in Homebrew packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - # - name: ccache - # uses: hendrikmuhs/ccache-action@v1.2 - # with: - # key: macos-13-xlarge - # max-size: 5G - # append-timestamp: true - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# # - name: ccache +# # uses: hendrikmuhs/ccache-action@v1.2 +# # with: +# # key: macos-13-xlarge +# # max-size: 5G +# # append-timestamp: true +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - # - name: Keep killall ccache and wait for ccache to finish - # if: always() - # run: | - # sleep 60 - # while ps -ef | grep ccache | grep -v grep; do \ - # killall ccache || true; \ - # sleep 10; \ - # done - # - name: Check ccache statistics - # run: | - # ccache -s || echo "ccache not available" - # ls -lh chdb - # df -h - # env: - # CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++" - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl - - name: Setup core dump - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-arm64.tar.gz" - ls -lh core-files-macos-arm64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-arm64 - path: core-files-macos-arm64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-arm64 - path: | - ./dist/*.whl - ./macos-arm64-libchdb.tar.gz - ./macos-arm64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# # - name: Keep killall ccache and wait for ccache to finish +# # if: always() +# # run: | +# # sleep 60 +# # while ps -ef | grep ccache | grep -v grep; do \ +# # killall ccache || true; \ +# # sleep 10; \ +# # done +# # - name: Check ccache statistics +# # run: | +# # ccache -s || echo "ccache not available" +# # ls -lh chdb +# # df -h +# # env: +# # CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++" +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl +# - name: Setup core dump +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-arm64.tar.gz" +# ls -lh core-files-macos-arm64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-arm64 +# path: core-files-macos-arm64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-arm64 +# path: | +# ./dist/*.whl +# ./macos-arm64-libchdb.tar.gz +# ./macos-arm64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index 6390a6849fc..0911a9b2757 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -1,318 +1,318 @@ -name: Build macOS X86 +# name: Build macOS X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS x86_64) - runs-on: macos-14-large - steps: - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS x86_64) +# runs-on: macos-14-large +# steps: +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext gcc binutils grep findutils nasm - brew install ccache || echo "ccache installation failed, continuing without it" - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$PATH - which clang++ - clang++ --version - which go - go version - ccache -s - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext gcc binutils grep findutils nasm +# brew install ccache || echo "ccache installation failed, continuing without it" +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$PATH +# which clang++ +# clang++ --version +# which go +# go version +# ccache -s +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in Homebrew packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/usr/local --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in Homebrew packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/usr/local --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: macos-14-x86_64 - max-size: 5G - append-timestamp: true - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: macos-14-x86_64 +# max-size: 5G +# append-timestamp: true +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - env: - CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++" - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl - - name: Setup core dump collection - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-x86_64.tar.gz" - ls -lh core-files-macos-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-x86_64 - path: core-files-macos-x86_64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-x86_64 - path: | - ./dist/*.whl - ./macos-x86_64-libchdb.tar.gz - ./macos-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# env: +# CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++" +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-x86_64.tar.gz" +# ls -lh core-files-macos-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-x86_64 +# path: core-files-macos-x86_64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-x86_64 +# path: | +# ./dist/*.whl +# ./macos-x86_64-libchdb.tar.gz +# ./macos-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml new file mode 100644 index 00000000000..db6db9d765d --- /dev/null +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -0,0 +1,269 @@ +name: Build Linux(musllinux) x86 + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + + +jobs: + build_musllinux_wheels: + name: Build musllinux wheels (Alpine Linux x86_64) + runs-on: gh-64c + container: + image: quay.io/pypa/musllinux_1_2_x86_64 + options: --privileged + steps: + - name: Check system info + run: | + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + if [ -f /lib/ld-musl-x86_64.so.1 ]; then + echo "musl libc x86_64" + elif [ -f /lib/libc.musl-x86_64.so.1 ]; then + echo "musl libc x86_64" + else + echo "Not musl libc" + fi + - name: Install Python build dependencies + run: | + apk update + apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers + - name: Scan SQLite vulnerabilities with grype + run: | + # Install grype and required tools + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + + # Update grype vulnerability database + grype db update + + # Check SQLite vulnerabilities in installed packages + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + else + echo "No SQLite vulnerabilities found" + fi + continue-on-error: false + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv install 3.8:latest + pyenv install 3.9:latest + # pyenv install 3.10:latest + # pyenv install 3.11:latest + # pyenv install 3.12:latest + # pyenv install 3.13:latest + # pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + pyenv global 3.8 3.9 + + # Verify installations + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Installed Python versions:" + pyenv versions + echo "" + echo "Verifying all required Python versions are available:" + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.8 3.9; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "✓ Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.8 3.9; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset + done + - name: Install clang++ for Alpine + run: | + apk add --no-cache make cmake ccache ninja yasm gawk wget + apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + # Install Rust toolchain via rustup for proper target management + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + # Add the correct musl target for Alpine + rustup target add x86_64-unknown-linux-musl + rustc --version + cargo --version + ccache -s + - name: Update git + run: | + apk add --no-cache git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Configure git safe directory + run: | + git config --global --add safe.directory '*' + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: musllinux-1-2-x86_64 + max-size: 5G + append-timestamp: true + - name: setup clang and link clang-20 to clang + run: | + ln -sf /usr/bin/clang-20 /usr/bin/clang + ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + # Setup Rust environment + source $HOME/.cargo/env + pyenv shell 3.8 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build.sh + pyenv shell 3.8 + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." + + # Files to scan + FILES_TO_SCAN="" + [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" + [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + + SQLITE_VULNERABILITIES_FOUND=false + + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" + else + echo "No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + cp bin/patchelf /usr/bin/ + chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + rm -f dist/*-linux_x86_64.whl + ls -lh dist + shell: bash + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.9; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-musllinux-x86_64 + path: | + ./dist/*.whl + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/chdb/build_pybind11.sh b/chdb/build_pybind11.sh index ccc6eaa7602..cb7074cd57c 100755 --- a/chdb/build_pybind11.sh +++ b/chdb/build_pybind11.sh @@ -63,7 +63,8 @@ build_pybind11_nonlimitedapi() { } build_all_pybind11_nonlimitedapi() { - local python_versions=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13") + # local python_versions=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13") + local python_versions=("3.8" "3.9") echo "Building pybind11 nonlimitedapi libraries for all Python versions..." diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index fa58a739611..ddf9d940e15 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -156,20 +156,20 @@ elseif (ARCH_AMD64) endif() # Same best-effort check for x86 as above for ARM. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) - # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. - # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse - # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf - # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c - # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx - # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" - execute_process( - COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) - if (NOT FLAGS) - MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") - endif() - endif() + # if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) + # # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. + # # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse + # # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf + # # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c + # # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx + # # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" + # execute_process( + # COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo + # OUTPUT_VARIABLE FLAGS) + # if (NOT FLAGS) + # MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") + # endif() + # endif() # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/ # AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary.