mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 22:35:42 +00:00
Fix scheduled cleanup workflows (#7218)
This commit is contained in:
@@ -14,7 +14,21 @@ jobs:
|
||||
runs-on: [self-hosted, bazel]
|
||||
|
||||
steps:
|
||||
- name: Prepare non-LFS checkout
|
||||
env:
|
||||
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
|
||||
GIT_CONFIG_NOSYSTEM: "1"
|
||||
run: |
|
||||
git config --global --unset-all filter.lfs.process || true
|
||||
git config --global filter.lfs.smudge "cat"
|
||||
git config --global filter.lfs.clean "cat"
|
||||
git config --global filter.lfs.required false
|
||||
rm -f .git/hooks/post-checkout .git/hooks/post-merge .git/hooks/pre-push
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
env:
|
||||
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
|
||||
GIT_CONFIG_NOSYSTEM: "1"
|
||||
with:
|
||||
persist-credentials: false
|
||||
lfs: false
|
||||
|
||||
@@ -14,73 +14,32 @@ jobs:
|
||||
runs-on: [self-hosted, bazel]
|
||||
|
||||
steps:
|
||||
- name: Ensure AWS CLI is available
|
||||
- name: Prepare non-LFS checkout
|
||||
env:
|
||||
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
|
||||
GIT_CONFIG_NOSYSTEM: "1"
|
||||
run: |
|
||||
if ! command -v aws &> /dev/null; then
|
||||
brew install awscli
|
||||
fi
|
||||
git config --global --unset-all filter.lfs.process || true
|
||||
git config --global filter.lfs.smudge "cat"
|
||||
git config --global filter.lfs.clean "cat"
|
||||
git config --global filter.lfs.required false
|
||||
rm -f .git/hooks/post-checkout .git/hooks/post-merge .git/hooks/pre-push
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
env:
|
||||
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig-no-lfs
|
||||
GIT_CONFIG_NOSYSTEM: "1"
|
||||
with:
|
||||
persist-credentials: false
|
||||
lfs: false
|
||||
clean: false
|
||||
|
||||
- name: Ensure Bazel installed
|
||||
run: ./ci/github_actions/ensure_bazel_installed.sh
|
||||
|
||||
- name: Delete old archived game folders
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.DO_SPACES_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACES_SECRET_KEY }}
|
||||
DO_SPACES_ACCESS_KEY: ${{ secrets.DO_SPACES_ACCESS_KEY }}
|
||||
DO_SPACES_SECRET_KEY: ${{ secrets.DO_SPACES_SECRET_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
S3_ENDPOINT="https://sfo3.digitaloceanspaces.com"
|
||||
BUCKET="s3://eagle0/eagle/archived/"
|
||||
|
||||
# macOS date syntax
|
||||
CUTOFF=$(date -v-1m +%s)
|
||||
DELETED=0
|
||||
SKIPPED=0
|
||||
NO_DIR_FILE=0
|
||||
|
||||
echo "Cutoff date: $(date -r ${CUTOFF} '+%Y-%m-%dT%H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
FOLDERS=$(aws s3 ls "$BUCKET" --endpoint-url "$S3_ENDPOINT" \
|
||||
| awk '/PRE/{gsub(/\/$/,"",$2); print $2}')
|
||||
|
||||
if [ -z "$FOLDERS" ]; then
|
||||
echo "No archived folders found."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TOTAL=$(echo "$FOLDERS" | wc -l | tr -d ' ')
|
||||
CURRENT=0
|
||||
|
||||
for game_id in $FOLDERS; do
|
||||
CURRENT=$((CURRENT + 1))
|
||||
|
||||
DIR_INFO=$(aws s3 ls "${BUCKET}${game_id}/directory.e0i" \
|
||||
--endpoint-url "$S3_ENDPOINT" 2>/dev/null || true)
|
||||
|
||||
if [ -z "$DIR_INFO" ]; then
|
||||
echo "[$CURRENT/$TOTAL] DELETE $game_id (no directory.e0i)"
|
||||
aws s3 rm --recursive "${BUCKET}${game_id}/" \
|
||||
--endpoint-url "$S3_ENDPOINT" > /dev/null 2>&1
|
||||
DELETED=$((DELETED + 1))
|
||||
NO_DIR_FILE=$((NO_DIR_FILE + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
FILE_DATE=$(echo "$DIR_INFO" | awk '{print $1 " " $2}')
|
||||
FILE_EPOCH=$(date -j -f '%Y-%m-%d %H:%M:%S' "$FILE_DATE" +%s 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$FILE_EPOCH" -lt "$CUTOFF" ]; then
|
||||
echo "[$CURRENT/$TOTAL] DELETE $game_id (directory.e0i from $FILE_DATE)"
|
||||
aws s3 rm --recursive "${BUCKET}${game_id}/" \
|
||||
--endpoint-url "$S3_ENDPOINT" > /dev/null 2>&1
|
||||
DELETED=$((DELETED + 1))
|
||||
else
|
||||
echo "[$CURRENT/$TOTAL] KEEP $game_id (directory.e0i from $FILE_DATE)"
|
||||
SKIPPED=$((SKIPPED + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
echo "Total folders: $TOTAL"
|
||||
echo "Deleted: $DELETED"
|
||||
echo "Kept (recent): $SKIPPED"
|
||||
bazel run //src/main/go/net/eagle0/build/s3_archive_cleanup:s3_archive_cleanup -- --min-age=720h
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
|
||||
|
||||
go_library(
|
||||
name = "s3_archive_cleanup_lib",
|
||||
srcs = ["main.go"],
|
||||
importpath = "github.com/nolen777/eagle0/src/main/go/net/eagle0/build/s3_archive_cleanup",
|
||||
visibility = ["//visibility:private"],
|
||||
deps = ["//src/main/go/net/eagle0/util/aws"],
|
||||
)
|
||||
|
||||
go_binary(
|
||||
name = "s3_archive_cleanup",
|
||||
embed = [":s3_archive_cleanup_lib"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nolen777/eagle0/src/main/go/net/eagle0/util/aws"
|
||||
)
|
||||
|
||||
const (
|
||||
bucketName = "eagle0"
|
||||
archivePrefix = "eagle/archived/"
|
||||
directoryFile = "directory.e0i"
|
||||
)
|
||||
|
||||
func main() {
|
||||
minAge := flag.Duration("min-age", 30*24*time.Hour, "Delete archives whose directory.e0i is older than this duration")
|
||||
dryRun := flag.Bool("dry-run", false, "Print what would be deleted without actually deleting")
|
||||
flag.Parse()
|
||||
|
||||
bb, err := aws.NewBucketBasics()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create AWS client: %v", err)
|
||||
}
|
||||
|
||||
keys, err := bb.ListObjectKeys(bucketName, archivePrefix)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to list s3://%s/%s: %v", bucketName, archivePrefix, err)
|
||||
}
|
||||
|
||||
archives := groupArchiveKeys(keys)
|
||||
if len(archives) == 0 {
|
||||
log.Println("No archived folders found.")
|
||||
return
|
||||
}
|
||||
|
||||
cutoff := time.Now().Add(-*minAge)
|
||||
log.Printf("Cutoff date: %s", cutoff.Format(time.RFC3339))
|
||||
log.Printf("Found %d archived folders", len(archives))
|
||||
|
||||
gameIDs := make([]string, 0, len(archives))
|
||||
for gameID := range archives {
|
||||
gameIDs = append(gameIDs, gameID)
|
||||
}
|
||||
sort.Strings(gameIDs)
|
||||
|
||||
deleted := 0
|
||||
skipped := 0
|
||||
missingDirectoryFile := 0
|
||||
for index, gameID := range gameIDs {
|
||||
keys := archives[gameID]
|
||||
directoryKey := archivePrefix + gameID + "/" + directoryFile
|
||||
lastModified, hasDirectoryFile, err := directoryLastModified(bb, keys, directoryKey)
|
||||
if err != nil {
|
||||
log.Printf("[%d/%d] SKIP %s (could not inspect %s: %v)", index+1, len(gameIDs), gameID, directoryFile, err)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
shouldDelete := !hasDirectoryFile || lastModified.Before(cutoff)
|
||||
if !shouldDelete {
|
||||
log.Printf("[%d/%d] KEEP %s (%s from %s)", index+1, len(gameIDs), gameID, directoryFile, lastModified.Format(time.RFC3339))
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
reason := fmt.Sprintf("%s from %s", directoryFile, lastModified.Format(time.RFC3339))
|
||||
if !hasDirectoryFile {
|
||||
reason = "no " + directoryFile
|
||||
missingDirectoryFile++
|
||||
}
|
||||
log.Printf("[%d/%d] DELETE %s (%s)", index+1, len(gameIDs), gameID, reason)
|
||||
|
||||
if *dryRun {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
if err := bb.DeleteObject(bucketName, key); err != nil {
|
||||
log.Printf("Warning: failed to delete %s: %v", key, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
action := "Deleted"
|
||||
if *dryRun {
|
||||
action = "Would delete"
|
||||
}
|
||||
log.Printf("=== Summary ===")
|
||||
log.Printf("Total folders: %d", len(archives))
|
||||
log.Printf("%s: %d", action, deleted)
|
||||
log.Printf("Kept or skipped: %d", skipped)
|
||||
log.Printf("Deleted due to missing %s: %d", directoryFile, missingDirectoryFile)
|
||||
}
|
||||
|
||||
func groupArchiveKeys(keys []string) map[string][]string {
|
||||
archives := make(map[string][]string)
|
||||
for _, key := range keys {
|
||||
relativeKey := strings.TrimPrefix(key, archivePrefix)
|
||||
parts := strings.SplitN(relativeKey, "/", 2)
|
||||
if len(parts) < 2 || parts[0] == "" {
|
||||
continue
|
||||
}
|
||||
archives[parts[0]] = append(archives[parts[0]], key)
|
||||
}
|
||||
return archives
|
||||
}
|
||||
|
||||
func directoryLastModified(bb aws.BucketBasics, keys []string, directoryKey string) (time.Time, bool, error) {
|
||||
for _, key := range keys {
|
||||
if key != directoryKey {
|
||||
continue
|
||||
}
|
||||
lastModified, err := bb.GetObjectLastModified(bucketName, directoryKey)
|
||||
return lastModified, true, err
|
||||
}
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
Reference in New Issue
Block a user