neodb #39
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/neodb.yml | |
| # ref: https://github.com/eallion/eallion.com/blob/main/.github/workflows/sync-neodb-data.yml | |
| name: neodb | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| watch: | |
| types: [started] | |
| # workflow_dispatch: | |
| # repository_dispatch: | |
| # types: [sync-neodb] | |
| jobs: | |
| synch_neodb: | |
| name: Sync NeoDB Data | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # 检查是否安装了 JQ | |
| - name: Check JQ | |
| run: | | |
| if ! command -v jq &> /dev/null; then | |
| echo "jq is not installed. Installing..." | |
| sudo apt-get update | |
| sudo apt-get install -y jq | |
| else | |
| echo "jq is already installed." | |
| fi | |
| # 把当前目录保存到环境变量中 | |
| echo "WORK_DIR=$(pwd)" >> $GITHUB_ENV | |
| # 获取本地现有文件中最新的 UUID | |
| - name: Get Local Current UUID | |
| run: | | |
| CURRENT_UUID() { | |
| jq -r '.data[0].item.uuid' _data/neodb/neodb_data.json | |
| } | |
| echo "CURRENT_UUID=$(CURRENT_UUID)" >> $GITHUB_ENV | |
| # 获取远程最新的 UUID | |
| - name: Get NeoDB Latest UUID and Pages | |
| run: | | |
| RESPONSE=$(curl -s -X 'GET' \ | |
| 'https://neodb.social/api/me/shelf/complete?page=1' \ | |
| -H 'accept: application/json' \ | |
| -H 'Authorization: Bearer ${{ secrets.NEODB_ACCESS_TOKEN }}') | |
| REMOTE_UUID=$(echo "$RESPONSE" | jq -r '.data[0].item.uuid') | |
| REMOTE_PAGES=$(echo "$RESPONSE" | jq -r '.pages') | |
| echo "REMOTE_UUID=$REMOTE_UUID" >> $GITHUB_ENV | |
| echo "REMOTE_PAGES=$REMOTE_PAGES" >> $GITHUB_ENV | |
| # 对比本地的 UUID 和远程 UUID，相等就跳过，不相等就下载新数据 | |
| - name: UUID Compare | |
| run: | | |
| if [ ${{ env.REMOTE_UUID }} != ${{ env.CURRENT_UUID }} ]; then | |
| echo "Variables are not equal. Running the next steps." | |
| else | |
| echo "Variables are equal. Skipping the next steps." | |
| exit 0 | |
| fi | |
| # 下载所有数据 | |
| - name: Get All NeoDB | |
| if: ${{ env.REMOTE_UUID != env.CURRENT_UUID }} | |
| run: | | |
| pages=${{ env.REMOTE_PAGES }} | |
| mkdir -p ${{ env.WORK_DIR }}/_data/neodb | |
| # 个人使用，新建 WorkDIR ，排除 vercel.json 和 package.json 等 | |
| mkdir neodb | |
| cd neodb | |
| # 下载数据 | |
| for ((i=1; i<=$pages; i++)); do | |
| url="https://neodb.social/api/me/shelf/complete?page=$i" | |
| filename="neodb_data$i.json" | |
| # 下载文件并保存为对应的文件名 | |
| curl -X 'GET' "$url" \ | |
| -H 'accept: application/json' \ | |
| -H 'Authorization: Bearer ${{ secrets.NEODB_ACCESS_TOKEN }}' > "$filename" | |
| done | |
| # 把所有数据合并成一个文件 | |
| jq -c -s '{data: map(.data[]) | unique | sort_by(.created_time) | reverse, pages: map(.pages)[0], count: map(.count)[0]}' *.json > neodb_data.json | |
| # 更新 NeoDB 数据 | |
| mkdir -p ${{ env.WORK_DIR }}/_data/neodb/ | |
| cp -f neodb_data.json ${{ env.WORK_DIR }}/_data/neodb/ | |
| cd .. | |
| # 按 category 分类 JSON 数据 | |
| - name: Categorize JSON by Category | |
| if: ${{ env.REMOTE_UUID != env.CURRENT_UUID }} | |
| run: | | |
| # 创建输出目录 | |
| mkdir -p ${{ env.WORK_DIR }}/_data/neodb | |
| # 提取所有 category 值 | |
| categories=$(jq -r '.data[].item.category' ${{ env.WORK_DIR }}/_data/neodb/neodb_data.json | tr -d '"' | sort | uniq) | |
| # categories=$(jq -r '.data[].item.category | unique[]' ${{ env.WORK_DIR }}/_data/neodb/neodb_data.json) | |
| # 按 category 分类并保存为单独的文件 | |
| for category in $categories; do | |
| jq --arg cat "$category" '{data: [.data[] | select(.item.category == $cat)]}' ${{ env.WORK_DIR }}/_data/neodb/neodb_data.json > ${{ env.WORK_DIR }}/_data/neodb/${category}.json | |
| done | |
| echo "JSON 数据已按 category 分类！" | |
| - name: Download NeoDB Cover | |
| if: ${{ env.REMOTE_UUID != env.CURRENT_UUID }} | |
| run: | | |
| # 检查 neodb_data 目录是否存在，如果不存在则创建 | |
| if [ ! -d "neodb_data" ]; then | |
| mkdir neodb_data | |
| fi | |
| # 读取本地的 neodb_data.json 文件内容 | |
| json=$(cat _data/neodb/neodb_data.json) | |
| # 提取图片 URL | |
| image_urls=$(echo "$json" | jq -r '.data[].item.cover_image_url') | |
| # 遍历图片 URL 并下载图片 | |
| for url in $image_urls; do | |
| filename=$(basename "$url") | |
| filepath="${{ env.WORK_DIR }}/assets/images/neodb/$filename" | |
| # 检查文件是否已存在 | |
| if [ -f "$filepath" ]; then | |
| echo "Skipping $filename - File already exists" | |
| else | |
| # 使用 curl 命令下载图片 | |
| curl -o "$filepath" "$url" | |
| echo "Downloaded $filename" | |
| echo "REMOTE_UUID=''" >> $GITHUB_ENV | |
| fi | |
| done | |
| # 把修改后的数据提交到 GitHub 仓库 | |
| - name: Git Add and Commit | |
| if: ${{ env.REMOTE_UUID != env.CURRENT_UUID }} | |
| uses: EndBug/add-and-commit@v9 | |
| with: | |
| message: 'chore(data): update neodb data' | |
| committer_name: 'github-actions[bot]' | |
| committer_email: 'github-actions[bot]@users.noreply.github.com' | |
| add: | | |
| './_data/neodb' | |
| './assets/images/neodb' |