Sidebar tabs #242
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: backend checks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - "backend/**" | |
| - ".github/backend_checks.yml" | |
| - "docker/**" | |
| - "tools/**" | |
| jobs: | |
| backend-checks: | |
| runs-on: self-hosted | |
| env: | |
| API_WORKERS: 1 | |
| RAY_HAS_CHANGED: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Checkout PR | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh pr checkout ${{ github.event.pull_request.number }} | |
| - name: Check for changes in ray_model_worker (pull request) | |
| id: check_ray_model_worker_changes_pr | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD) | |
| if git diff --name-only $base_branch HEAD | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/'; then | |
| echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV | |
| fi | |
| - name: Check for changes in ray_model_worker (push) | |
| id: check_ray_model_worker_changes_push | |
| if: github.event_name == 'push' | |
| run: | | |
| if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/'; then | |
| echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV | |
| fi | |
| - name: Prepare environment | |
| run: | | |
| ./bin/setup-folders.sh | |
| ./bin/setup-envs.sh --project_name action-runner --port_prefix 131 | |
| sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env | |
| echo "MODELS_CACHE_DIR=$HOME/models_cache" >> docker/.env | |
| mkdir -p $HOME/models_cache | |
| if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then | |
| sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env | |
| sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.ray.yml | |
| sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.yml | |
| echo "SPACY_CACHE_DIR=$HOME/spacy_models" >> docker/.env | |
| mkdir -p $HOME/spacy_models | |
| fi | |
| - name: Build & Start Docker Containers | |
| working-directory: docker | |
| run: | | |
| if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then | |
| docker compose -f compose.ray.yml build | |
| docker compose -f compose.ray.yml up --wait | |
| fi | |
| COMPOSE_PROFILES="weaviate,background,backend" docker compose build | |
| COMPOSE_PROFILES="weaviate,background" docker compose up --wait --quiet-pull | |
| - name: Check 1 - pytest runs without errors | |
| working-directory: docker | |
| run: | | |
| docker compose run dats-backend-api /opt/envs/dats/bin/python -m pytest | |
| - name: Check 2 - Database migrates without errors database | |
| working-directory: docker | |
| run: | | |
| docker compose run -e PYTHONPATH='/dats_code/src' dats-backend-api /opt/envs/dats/bin/python migration/run_migrations.py | |
| - name: Check 3 - Database schema is up-to-date after migration | |
| working-directory: docker | |
| run: | | |
| docker compose run dats-backend-api /opt/envs/dats/bin/alembic check | |
| - name: Start Remaining Docker Containers | |
| working-directory: docker | |
| run: | | |
| COMPOSE_PROFILES="weaviate,background,backend" docker compose up --wait --quiet-pull | |
| - name: Check 4 - Test End-2-End importer script | |
| working-directory: tools/importer | |
| env: | |
| TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }} | |
| run: | | |
| pip install -r requirements.txt | |
| wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip | |
| unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip | |
| python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text | |
| python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image | |
| - name: Check 5 - pyright runs without errors | |
| run: | | |
| micromamba env create -f backend/environment.yml --yes | |
| micromamba run -n dats pip install -r backend/src/app/preprocessing/ray_model_worker/requirements.txt | |
| micromamba run -n dats pip install ray==2.32.0 | |
| micromamba run -n dats pyright | |
| - name: Cleanup | |
| working-directory: docker | |
| if: always() | |
| run: | | |
| if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then | |
| docker compose -f compose.ray.yml down -v | |
| RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env) | |
| docker rmi $RAY_IMAGE | |
| fi | |
| docker compose down -v --remove-orphans | |
| micromamba env remove -n dats --yes | |
| BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env) | |
| docker rmi $BACKEND_IMAGE |