admin/eagle0
1
0
Fork 0
mirror of https://github.com/nolen777/eagle0.git synced 2026-07-28 20:55:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upload game assets to DigitalOcean too (#4056)

Browse Source 
* upload the game assets to DOS too

* unused deps

* handle missing manifest and read keys from config

* build and upload unity build on pull request

* try reading from env variable

* bypass

* just look at lengths

* bad env name

* restore the build

* but not on PRs

* Put S3 info into config file and point it to DigitalOcean
This commit is contained in:
admin
2025-03-22 21:39:21 -07:00
committed by GitHub
parent 4b17497893
commit aec0481e43
11 changed files with 155 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/unity_build.yml
+3
View File
@@ -36,6 +36,9 @@ jobs:
- name: Persist Library/
run: ./ci/github_actions/persist_library.sh
- name: Deploy Windows unity
env:
ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }}
SECRET_KEY: ${{ secrets.SECRET_KEY }}
run: bazel run //src/main/go/net/eagle0/build/unity3d_windows_build_handler:unity3d_windows_build_handler -- "/tmp/eagle0/eagle0WIN"
- name: Archive build log
if: success() || failure()
MODULE.bazel
+14 -14
View File
@@ -1,4 +1,4 @@
bazel_dep(name = "apple_support", repo_name = "build_bazel_apple_support", version = "1.15.1")
bazel_dep(name = "apple_support", version = "1.15.1", repo_name = "build_bazel_apple_support")
#
# bazel-toolchain
@@ -8,12 +8,10 @@ bazel_dep(name = "toolchains_llvm", version = "1.2.0")
# Configure and register the toolchain.
llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm")
llvm.toolchain(
name = "llvm_toolchain",
llvm_version = "19.1.0",
)
use_repo(llvm, "llvm_toolchain")
# Set dev_dependency so we can turn this off for swift MacOS builds
@@ -24,22 +22,26 @@ register_toolchains(
bazel_dep(name = "rules_pkg", version = "1.0.1")
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "protobuf", repo_name = "com_google_protobuf", version = "29.2")
bazel_dep(name = "protobuf", version = "29.2", repo_name = "com_google_protobuf")
bazel_dep(name = "grpc", version = "1.69.0")
bazel_dep(name = "grpc-java", version = "1.69.0")
bazel_dep(name = "googletest", version = "1.15.2")
bazel_dep(name = "rules_go", repo_name = "io_bazel_rules_go", version = "0.50.1")
bazel_dep(name = "gazelle", repo_name = "bazel_gazelle", version = "0.40.0")
bazel_dep(name = "rules_go", version = "0.50.1", repo_name = "io_bazel_rules_go")
bazel_dep(name = "gazelle", version = "0.40.0", repo_name = "bazel_gazelle")
go_sdk = use_extension("@io_bazel_rules_go//go:extensions.bzl", "go_sdk")
go_sdk.download(version = "1.23.3")
go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps")
go_deps.from_file(go_mod = "//:go.mod")
use_repo(go_deps, "com_github_aws_aws_sdk_go_v2", "com_github_aws_aws_sdk_go_v2_config", "com_github_aws_aws_sdk_go_v2_service_s3", "org_golang_google_protobuf")
use_repo(
go_deps,
"com_github_aws_aws_sdk_go_v2",
"com_github_aws_aws_sdk_go_v2_config",
"com_github_aws_aws_sdk_go_v2_credentials",
"com_github_aws_aws_sdk_go_v2_service_s3",
"org_golang_google_protobuf",
)
#go_sdk.nogo(
# nogo = "//:my_nogo",
@@ -57,7 +59,6 @@ bazel_dep(
)
maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
maven.install(
artifacts = [
"org.scala-lang:scala-library:%s" % scala_version,
@@ -103,7 +104,6 @@ maven.install(
"https://repo1.maven.org/maven2",
],
)
use_repo(maven, "maven", "unpinned_maven")
#
@@ -112,13 +112,13 @@ use_repo(maven, "maven", "unpinned_maven")
bazel_dep(
name = "rules_apple",
repo_name = "build_bazel_rules_apple",
version = "3.16.1",
repo_name = "build_bazel_rules_apple",
)
bazel_dep(
name = "rules_swift",
repo_name = "build_bazel_rules_swift",
version = "2.3.1",
repo_name = "build_bazel_rules_swift",
)
#
go.mod
+1 -1
View File
@@ -6,12 +6,12 @@ require (
github.com/aws/aws-sdk-go-v2 v1.32.8
github.com/aws/aws-sdk-go-v2/config v1.28.10
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2
github.com/aws/aws-sdk-go-v2/credentials v1.17.51
google.golang.org/protobuf v1.36.3
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.51 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 // indirect
src/main/csharp/net/eagle0/clients/win/installer/EagleInstaller.sln.DotSettings.user
+3
View File
@@ -0,0 +1,3 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F8a221793e4b13ae450dccb6e74f6985a273b27d67dca5c4bc2c2e56df33bfe7_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileStream_002EWindows_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Ffc8f9bd3c0ac5ab6b9dabf421c55f1afeedd8abe1dfc288502292a552fb8d_003FFileStream_002EWindows_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
src/main/csharp/net/eagle0/clients/win/installer/EagleInstaller/EagleInstaller.csproj
+6
View File
@@ -14,4 +14,10 @@
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.8" />
</ItemGroup>
<ItemGroup>
<None Remove="configuration.txt" />
<Content Include="configuration.txt">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>
src/main/csharp/net/eagle0/clients/win/installer/EagleInstaller/EagleUpdater.cs
+61 -40
View File
@@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection;
using System.Threading.Tasks;
using System.Threading;
@@ -16,31 +17,59 @@ namespace EagleInstaller {
public UpdaterFailureException(string message) : base(message) {}
}
class EagleUpdater {
private static readonly string bucketName = "eagle0";
private static readonly string manifestName = "eagle0WIN_manifest.txt";
private static readonly string remoteManifestName = "unity3d/" + manifestName;
private static readonly string OLD_MANIFEST_PATH_PREFIX = "./eagle0WIN/";
class EagleUpdater
{
private static readonly Dictionary<string, string> ConfigFileEntries = ReadConfiguration();
private static readonly string ServiceUrl = ConfigFileEntries["service_url"];
private static readonly string BucketName = ConfigFileEntries["bucket_name"];
private static readonly string ManifestName = ConfigFileEntries["manifest_name"];
private static readonly string RemoteManifestName = ConfigFileEntries["remote_manifest_prefix"] + ManifestName;
private static readonly int DOWNLOAD_SLOTS = 4;
private static readonly int RETRY_COUNT = 3;
private const int DownloadSlots = 4;
private const int RetryCount = 3;
private static readonly string ACCESS_KEY_ID = "AKIAU575INTWMNURXP42";
private static readonly string SECRET_ACCESS_KEY =
"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";
private static readonly string REMOTE_PREFIX = "unity3d/win/";
private static readonly string AccessKeyId = ConfigFileEntries["access_key_id"];
private AmazonS3Client s3Client;
private static readonly string SecretAccessKey = ConfigFileEntries["secret_key"];
private static Semaphore _pool = new Semaphore(DOWNLOAD_SLOTS, DOWNLOAD_SLOTS);
private const string RemotePrefix = "unity3d/win/";
public struct FetchAttempt {
public bool success;
public KeyValuePair<string, string> kvp;
private readonly AmazonS3Client _s3Client;
private static Semaphore _pool = new Semaphore(DownloadSlots, DownloadSlots);
private struct FetchAttempt {
public bool Success;
public KeyValuePair<string, string> Kvp;
}
private static Dictionary<string, string> ReadConfiguration()
{
string path = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), @"configuration.txt");
var configText = File.ReadAllText(path);
if (string.IsNullOrEmpty(configText)) { return new Dictionary<string, string>(); }
var dict = new Dictionary<string, string>();
using (StringReader sr = new StringReader(configText)) {
string line;
while ((line = sr.ReadLine()) != null)
{
var components = line.Split(" = ");
dict.Add(components[0], components[1]);
}
}
return dict;
}
public EagleUpdater() {
s3Client = new AmazonS3Client(ACCESS_KEY_ID, SECRET_ACCESS_KEY, RegionEndpoint.USEast1);
AmazonS3Config config = new AmazonS3Config
{
ServiceURL = ServiceUrl
};
_s3Client = new AmazonS3Client(AccessKeyId, SecretAccessKey, config);
}
private List<string> Discrepancies(
@@ -54,7 +83,7 @@ namespace EagleInstaller {
}
public async Task<bool> MaybePerformUpdateWithRetries(string localPath) {
string existingManifestPath = Path.Combine(localPath, manifestName);
string existingManifestPath = Path.Combine(localPath, ManifestName);
Console.WriteLine("Loading old manifest");
string existingText = await LocalManifestText(existingManifestPath);
@@ -72,7 +101,7 @@ namespace EagleInstaller {
List<string> discrepancies = Discrepancies(desiredFiles, currentFiles);
for (int retry = 0; retry < RETRY_COUNT; retry++) {
for (int retry = 0; retry < RetryCount; retry++) {
Console.WriteLine($"Attempt {retry + 1}");
var newFiles = await AttemptUpdate(localPath, desiredFiles, currentFiles);
@@ -100,7 +129,7 @@ namespace EagleInstaller {
}
// sadness
Console.WriteLine($"Exhausted {RETRY_COUNT} attempts. Giving up.");
Console.WriteLine($"Exhausted {RetryCount} attempts. Giving up.");
return false;
}
@@ -145,12 +174,12 @@ namespace EagleInstaller {
var fetchAttemptResults = await Task.WhenAll(updateTasks);
// Add successful downloads to the existingFiles dictionary
fetchAttemptResults.Where(fa => fa.success)
.Select(fa => fa.kvp)
fetchAttemptResults.Where(fa => fa.Success)
.Select(fa => fa.Kvp)
.ToList()
.ForEach(fa => existingFiles[fa.Key] = fa.Value);
if (fetchAttemptResults.ToList().TrueForAll(fa => fa.success)) {
if (fetchAttemptResults.ToList().TrueForAll(fa => fa.Success)) {
Console.WriteLine("All download tasks succeeded.");
} else {
Console.WriteLine("Failed to fetch all files!");
@@ -175,26 +204,18 @@ namespace EagleInstaller {
}
async Task<string> RemoteManifestText() {
using (GetObjectResponse response = await s3Client.GetObjectAsync(bucketName, remoteManifestName)) using (
using (GetObjectResponse response = await _s3Client.GetObjectAsync(BucketName, RemoteManifestName)) using (
StreamReader reader = new StreamReader(response.ResponseStream)) {
return await reader.ReadToEndAsync();
}
}
static String StripOldPrefix(string path) {
if (path.StartsWith(OLD_MANIFEST_PATH_PREFIX)) {
return path.Substring(OLD_MANIFEST_PATH_PREFIX.Length);
} else {
return path;
}
}
static Dictionary<String, String> ShasFromText(string text) {
if (text == null || text.Length == 0) { return new Dictionary<string, string>(); }
List<string> lines = text.Split('\n').Where(str => str.Length > 0).ToList();
return lines.Select(str => str.Split(" "))
.ToDictionary(strs => StripOldPrefix(strs[1]), strs => strs[0]);
.ToDictionary(strs => strs[1], strs => strs[0]);
}
static string TextFromShas(Dictionary<string, string> shas) {
@@ -222,7 +243,7 @@ namespace EagleInstaller {
_pool.WaitOne();
try {
using (GetObjectResponse response = await s3Client.GetObjectAsync(bucketName, REMOTE_PREFIX + remotePath)) using (
using (GetObjectResponse response = await _s3Client.GetObjectAsync(BucketName, RemotePrefix + remotePath)) using (
Stream responseStream =
response.ResponseStream) using (FileStream fileStream = File.Create(writePath)) using (MemoryStream memStream = new MemoryStream()) {
responseStream.CopyTo(memStream);
@@ -240,23 +261,23 @@ namespace EagleInstaller {
memStream.Position = 0;
await memStream.CopyToAsync(fileStream);
return new FetchAttempt() {
success = true,
kvp = new KeyValuePair<String, String>(remotePath, computedSha)
Success = true,
Kvp = new KeyValuePair<String, String>(remotePath, computedSha)
};
} else {
Console.WriteLine($"SHAs do not match for {remotePath}");
return new FetchAttempt() { success = false };
return new FetchAttempt() { Success = false };
}
}
} catch (TaskCanceledException e) {
Console.WriteLine($"Caught ObjectDisposedException {e}");
return new FetchAttempt() { success = false };
return new FetchAttempt() { Success = false };
} catch (HttpRequestException e) {
Console.WriteLine($"Caught HttpRequestException {e}");
return new FetchAttempt() { success = false };
return new FetchAttempt() { Success = false };
} catch (Exception e) {
Console.WriteLine($"Caught Exception {e}");
return new FetchAttempt() { success = false };
return new FetchAttempt() { Success = false };
}
}
}
src/main/csharp/net/eagle0/clients/win/installer/EagleInstaller/Program.cs
+4 -4
View File
@@ -5,19 +5,19 @@ using System.Threading;
namespace EagleInstaller {
class Program {
private static string fileLocation = Path.Combine(
private static readonly string FileLocation = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"eagle0");
private static string exeLocation = Path.Combine(fileLocation, "eagle0.exe");
private static readonly string ExeLocation = Path.Combine(FileLocation, "eagle0.exe");
static async Task Main(string[] args) {
EagleUpdater updater = new EagleUpdater();
bool success = await updater.MaybePerformUpdateWithRetries(fileLocation);
bool success = await updater.MaybePerformUpdateWithRetries(FileLocation);
if (success) {
Console.WriteLine("Launching application");
System.Diagnostics.Process.Start(exeLocation);
System.Diagnostics.Process.Start(ExeLocation);
} else {
Console.WriteLine("Failed to update!");
Thread.Sleep(int.MaxValue);
src/main/csharp/net/eagle0/clients/win/installer/EagleInstaller/configuration.txt
+6
View File
@@ -0,0 +1,6 @@
service_url = https://sfo3.digitaloceanspaces.com
access_key_id = DO801YZPXXP88GULGNDQ
secret_key = 9FQhRoTWi3FlIG4hgHhz5l7sLnog0Sl10aVEQP1DOEM
bucket_name = eagle0-windows
manifest_name = eagle0WIN_manifest.txt
remote_manifest_prefix = unity3d/
src/main/go/net/eagle0/build/unity3d_windows_build_handler/unity3d_windows_build_handler.go
+2 -2
View File
@@ -12,7 +12,7 @@ import (
"strings"
)
var bucketName = "eagle0"
var bucketName = "eagle0-windows"
var manifestPath = "unity3d/eagle0WIN_manifest.txt"
var destinationRoot = "unity3d/win/"
@@ -146,7 +146,7 @@ func handleDir(tempDir string) {
remoteManifest, err := fetchRemoteManifest(bb)
if err != nil {
log.Fatal(err)
remoteManifest = ""
}
remoteShas := readShas(remoteManifest)
src/main/go/net/eagle0/util/aws/BUILD.bazel
+1 -1
View File
@@ -7,7 +7,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"@com_github_aws_aws_sdk_go_v2//aws",
"@com_github_aws_aws_sdk_go_v2_config//:config",
"@com_github_aws_aws_sdk_go_v2_credentials//:credentials",
"@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
"@com_github_aws_aws_sdk_go_v2_service_s3//types",
],
src/main/go/net/eagle0/util/aws/s3.go
+54 -10
View File
@@ -1,36 +1,80 @@
package aws
import (
"bufio"
"bytes"
"context"
"errors"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"io"
"log"
"os"
"strings"
"time"
)
// BucketBasics encapsulates the Amazon Simple Storage Service (Amazon S3) actions
// used in the examples.
// It contains S3Client, an Amazon S3 service client that is used to perform bucket
// and object actions.
type BucketBasics struct {
ctx context.Context
S3Client *s3.Client
}
func NewBucketBasics() (BucketBasics, error) {
ctx := context.Background()
sdkConfig, err := config.LoadDefaultConfig(ctx)
func ReadConfig() (map[string]string, error) {
homeDir, err := os.UserHomeDir()
if err != nil {
return BucketBasics{}, err
return nil, err
}
configFile, err := os.Open(homeDir + "/.s3cfg")
if err != nil {
return nil, err
}
s3Client := s3.NewFromConfig(sdkConfig)
scanner := bufio.NewScanner(configFile)
config := make(map[string]string)
config["key"] = "dummy_key"
config["secret"] = "dummy_secret"
for scanner.Scan() {
line := scanner.Text()
fields := strings.Split(line, " = ")
if len(fields) != 2 {
continue
}
config[fields[0]] = fields[1]
}
return config, nil
}
func NewBucketBasics() (BucketBasics, error) {
config, err := ReadConfig()
var key string
var secret string
if err == nil {
key = config["access_key"]
secret = config["secret_key"]
}
if err != nil {
key = os.Getenv("ACCESS_KEY_ID")
secret = os.Getenv("SECRET_KEY")
}
println("key size: ", len(key))
println("secret size: ", len(secret))
sdkConfig := aws.Config{
Credentials: credentials.NewStaticCredentialsProvider(key, secret, ""),
BaseEndpoint: aws.String("https://sfo3.digitaloceanspaces.com"),
Region: "us-east-1",
}
s3Client := s3.NewFromConfig(sdkConfig, func(o *s3.Options) {
o.UsePathStyle = false
})
ctx := context.Background()
return BucketBasics{
ctx: ctx,
S3Client: s3Client,