mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 20:55:44 +00:00
adminandGitHub
e08f327bd1
* remove all trailing whitespace * fix one scala file Former-commit-id: c73a66046649b189f5df617fefa6e099b1c05426
146 lines
4.8 KiB
Python
Executable File
146 lines
4.8 KiB
Python
Executable File
#/usr/bin/env python3
|
|
|
|
import boto3
|
|
import hashlib
|
|
import os
|
|
import sys
|
|
|
|
from botocore.exceptions import ClientError
|
|
|
|
bucket_name = 'eagle0'
|
|
|
|
def manifest_path(branch_name):
|
|
if branch_name == 'main':
|
|
return 'unity3d/eagle0WIN_manifest.txt'
|
|
else:
|
|
return 'unity3d/branches/' + branch_name + '/eagle0WIN_manifest.txt'
|
|
|
|
def destination_root(branch_name):
|
|
if branch_name == 'main':
|
|
return 'unity3d/win/'
|
|
else:
|
|
return 'unity3d/branches/' + branch_name + '/win/'
|
|
|
|
def generate_local_shas(local_dir):
|
|
sha_map = {}
|
|
|
|
for root, subdirs, files in os.walk(local_dir):
|
|
relRoot = os.path.relpath(root, local_dir)
|
|
|
|
for filename in files:
|
|
file_path = os.path.join(root, filename)
|
|
relative_path = os.path.relpath(file_path, local_dir)
|
|
|
|
with open(file_path, 'rb') as file_to_check:
|
|
# read contents of the file
|
|
data = file_to_check.read()
|
|
# pipe contents of the file through
|
|
sha_map[relative_path] = hashlib.sha256(data).hexdigest()
|
|
|
|
return sha_map
|
|
|
|
def generate_manifest(sha_map):
|
|
manifest = ''
|
|
|
|
for path, sha in sha_map.items():
|
|
manifest += sha + ' ' + path + '\n'
|
|
|
|
return manifest
|
|
|
|
def read_shas(manifest):
|
|
sha_map = {}
|
|
|
|
if manifest is not None:
|
|
for line in manifest.splitlines():
|
|
elts = line.split(' ')
|
|
sha_map[elts[1]] = elts[0]
|
|
|
|
return sha_map
|
|
|
|
def copy_to_branch(branch_name):
|
|
s3_client = boto3.client('s3')
|
|
|
|
print('copying manifest from main')
|
|
s3_client.copy_object(CopySource = bucket_name + '/' + manifest_path('main'), Bucket = bucket_name, Key = manifest_path(branch_name))
|
|
|
|
old_prefix = destination_root('main')
|
|
new_prefix = destination_root(branch_name)
|
|
|
|
main_objects = s3_client.list_objects_v2(Bucket = bucket_name, Prefix = old_prefix)
|
|
for obj in main_objects['Contents']:
|
|
source_key = obj['Key']
|
|
dest_key = source_key.replace(old_prefix, new_prefix, 1)
|
|
|
|
print('Copying ' + source_key + ' to ' + dest_key)
|
|
s3_client.copy_object(CopySource = bucket_name + '/' + source_key, Bucket = bucket_name, Key = dest_key)
|
|
|
|
|
|
def fetch_manifest(branch_name):
|
|
s3_client = boto3.client('s3')
|
|
|
|
try:
|
|
s3_response_object = s3_client.get_object(Bucket = bucket_name, Key = manifest_path(branch_name))
|
|
return s3_response_object['Body'].read().decode('UTF-8')
|
|
except ClientError as e:
|
|
print('No remote manifest found')
|
|
if branch_name != 'main':
|
|
print('Trying to copy from main')
|
|
copy_to_branch(branch_name)
|
|
return fetch_manifest(branch_name)
|
|
|
|
|
|
def syncToS3(local_dir, s3_root, local_shas, remote_shas):
|
|
s3_client = boto3.client('s3')
|
|
|
|
for root, subdirs, files in os.walk(local_dir):
|
|
relRoot = os.path.relpath(root, local_dir)
|
|
|
|
for filename in files:
|
|
file_path = os.path.join(root, filename)
|
|
relative_path = os.path.relpath(file_path, local_dir)
|
|
destination_path = os.path.join(s3_root, relative_path)
|
|
|
|
local_sha = local_shas[relative_path]
|
|
remote_sha = remote_shas.get(relative_path)
|
|
|
|
print(destination_path + ':')
|
|
|
|
object_matches = True
|
|
if (local_sha != remote_sha):
|
|
print('\tno match, uploading')
|
|
object_matches = False
|
|
elif (remote_sha is None):
|
|
print('\tno remote hash, uploading')
|
|
object_matches = False
|
|
else:
|
|
print('\tMatch, not uploading')
|
|
object_matches = True
|
|
# try:
|
|
# # print('trying to get for ' + destination_path + ' in ' + bucket_name)
|
|
# # attrs = s3_client.get_object_attributes(Bucket = bucket_name, Key = destination_path, ObjectAttributes=['ObjectSize'])
|
|
# # if attrs['ObjectSize'] != os.stat(file_path).st_size:
|
|
# # print('\tsizes don\'t match, uploading')
|
|
# # object_matches = False
|
|
|
|
# except ClientError as e:
|
|
# print('\tUnable to fetch remote file attributes: ' + str(e))
|
|
# object_matches = False
|
|
|
|
if not object_matches:
|
|
s3_client.upload_file(Bucket = bucket_name, Filename = file_path, Key = destination_path)
|
|
|
|
def handle_dir(tmp_dir, branch_name):
|
|
local_shas = generate_local_shas(tmp_dir)
|
|
local_manifest = generate_manifest(local_shas)
|
|
|
|
remote_manifest = fetch_manifest(branch_name)
|
|
|
|
remote_shas = read_shas(remote_manifest)
|
|
|
|
syncToS3(tmp_dir, destination_root(branch_name), local_shas, remote_shas)
|
|
|
|
s3_client = boto3.client('s3')
|
|
s3_client.put_object(Bucket = bucket_name, Body = local_manifest, Key = manifest_path(branch_name))
|
|
|
|
handle_dir(sys.argv[1], sys.argv[2])
|