Supports droping the masked tokens in MoE dispatching and fix some bugs #21774
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Codestyle Check | |
| on: [push, pull_request] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| PR_ID: ${{ github.event.pull_request.number }} | |
| COMMIT_ID: ${{ github.event.pull_request.head.sha }} | |
| BRANCH: ${{ github.event.pull_request.base.ref }} | |
| TASK: PaddleNLP-CI-Lint-${{ github.event.pull_request.number }} | |
| jobs: | |
| Lint: | |
| name: Lint | |
| runs-on: [self-hosted, ernie-cpu] | |
| steps: | |
| - name: Run Container | |
| env: | |
| work_dir: ${{ github.workspace }} | |
| python_version: "3.10" | |
| run: | | |
| container_name=${TASK}-$(date +%Y%m%d-%H%M%S) | |
| echo "container_name=${container_name}" >> ${{ github.env }} | |
| docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest" | |
| docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \ | |
| -v $work_dir/../../..:$work_dir/../../.. \ | |
| -v $work_dir:/workspace \ | |
| -v /home/.cache/pip:/home/.cache/pip \ | |
| -e BRANCH \ | |
| -e PR_ID \ | |
| -e COMMIT_ID \ | |
| -e work_dir \ | |
| -e no_proxy \ | |
| -e python_version \ | |
| -w /workspace ${docker_image} | |
| - name: Download Code | |
| env: | |
| work_dir: ${{ github.workspace }} | |
| run: | | |
| docker exec -t ${container_name} /bin/bash -c ' | |
| rm -rf * .[^.]* | |
| echo "Downloading PaddleNLP.tar" | |
| wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate | |
| echo "Extracting PaddleNLP.tar" | |
| tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar | |
| source $work_dir/../../../proxy | |
| cd PaddleNLP | |
| git config --global user.name "PaddleCI" | |
| git config --global user.email "paddle_ci@example.com" | |
| git pull | |
| git submodule update --init --recursive --force | |
| if [ -n "${PR_ID}" ]; then | |
| git fetch origin pull/${PR_ID}/head | |
| git checkout -b PR_${PR_ID} FETCH_HEAD | |
| git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git | |
| git fetch upstream ${BRANCH} | |
| git merge ${BRANCH} --no-edit | |
| git diff --numstat ${BRANCH} -- | awk "{print \$NF}" | |
| else | |
| echo "Not in a pull_request event. Skipping PR-specific operations." | |
| fi | |
| git log --pretty=oneline -10 | |
| if ! git show-ref --quiet refs/heads/develop; then \ | |
| echo "local develop branch is missing, creating local develop branch that tracks remote develop branch" | |
| git fetch origin develop | |
| git branch develop --track origin/develop | |
| else | |
| echo "local develop branch exist, skipping" | |
| fi | |
| unset http_proxy && unset https_proxy | |
| ' | |
| - name: Setup Environment | |
| run: | | |
| docker exec -t $container_name /bin/bash -c ' | |
| unlink /usr/local/bin/python | |
| ln -sf $(which python${python_version}) /usr/local/bin/python | |
| set -e | |
| python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| python -m pip config set global.cache-dir "/home/.cache/pip" | |
| python -m pip install --upgrade pip | |
| cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD | |
| make install | |
| ' | |
| - name: Test | |
| run: | | |
| docker exec -t $container_name /bin/bash -c ' | |
| set -e | |
| cd /workspace/PaddleNLP | |
| make lint | |
| ' | |
| - name: Terminate And Delete the Container | |
| if: always() | |
| run: | | |
| docker rm -f $container_name 2>/dev/null || true |