mirror of
https://github.com/nolen777/eagle0.git
synced 2026-07-28 20:55:44 +00:00
adminandGitHub
e08f327bd1
* remove all trailing whitespace * fix one scala file Former-commit-id: c73a66046649b189f5df617fefa6e099b1c05426
151 lines
4.7 KiB
Python
151 lines
4.7 KiB
Python
#/usr/bin/env python3
|
|
|
|
import boto3
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
from botocore.exceptions import ClientError
|
|
|
|
bucket_name = 'eagle0'
|
|
manifest_path = 'unity3d/eagle0WIN_manifest.txt'
|
|
destination_root = 'unity3d/win/'
|
|
|
|
def unzip(zip_path, unzip_dir):
|
|
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
|
zip_ref.extractall(unzip_dir)
|
|
|
|
def generate_local_shas(local_dir):
|
|
sha_map = {}
|
|
|
|
for root, subdirs, files in os.walk(local_dir):
|
|
relRoot = os.path.relpath(root, local_dir)
|
|
|
|
for filename in files:
|
|
file_path = os.path.join(root, filename)
|
|
relative_path = os.path.relpath(file_path, local_dir)
|
|
|
|
with open(file_path, 'rb') as file_to_check:
|
|
# read contents of the file
|
|
data = file_to_check.read()
|
|
# pipe contents of the file through
|
|
sha_map[relative_path] = hashlib.sha256(data).hexdigest()
|
|
|
|
return sha_map
|
|
|
|
def generate_manifest(sha_map):
|
|
manifest = ''
|
|
|
|
for path, sha in sha_map.items():
|
|
manifest += sha + ' ' + path + '\n'
|
|
|
|
return manifest
|
|
|
|
def read_shas(manifest):
|
|
sha_map = {}
|
|
|
|
if manifest is not None:
|
|
for line in manifest.splitlines():
|
|
elts = line.split(' ')
|
|
sha_map[elts[1]] = elts[0]
|
|
|
|
return sha_map
|
|
|
|
def fetch_manifest():
|
|
s3_client = boto3.client('s3')
|
|
|
|
try:
|
|
s3_response_object = s3_client.get_object(Bucket = bucket_name, Key = manifest_path)
|
|
return s3_response_object['Body'].read().decode('UTF-8')
|
|
except ClientError as e:
|
|
print('No remote manifest found')
|
|
return None
|
|
|
|
|
|
def syncToS3(local_dir, local_shas, remote_shas):
|
|
s3_client = boto3.client('s3')
|
|
|
|
for root, subdirs, files in os.walk(local_dir):
|
|
relRoot = os.path.relpath(root, local_dir)
|
|
|
|
for filename in files:
|
|
file_path = os.path.join(root, filename)
|
|
relative_path = os.path.relpath(file_path, local_dir)
|
|
destination_path = os.path.join(destination_root, relative_path)
|
|
|
|
local_sha = local_shas[relative_path]
|
|
remote_sha = remote_shas.get(relative_path)
|
|
|
|
print(destination_path + ':')
|
|
|
|
object_matches = True
|
|
if (local_sha != remote_sha):
|
|
print('\tno match, uploading')
|
|
object_matches = False
|
|
elif (remote_sha is None):
|
|
print('\tno remote hash, uploading')
|
|
object_matches = False
|
|
else:
|
|
print('\tMatch, not uploading')
|
|
object_matches = True
|
|
# try:
|
|
# # print('trying to get for ' + destination_path + ' in ' + bucket_name)
|
|
# # attrs = s3_client.get_object_attributes(Bucket = bucket_name, Key = destination_path, ObjectAttributes=['ObjectSize'])
|
|
# # if attrs['ObjectSize'] != os.stat(file_path).st_size:
|
|
# # print('\tsizes don\'t match, uploading')
|
|
# # object_matches = False
|
|
|
|
# except ClientError as e:
|
|
# print('\tUnable to fetch remote file attributes: ' + str(e))
|
|
# object_matches = False
|
|
|
|
if not object_matches:
|
|
s3_client.upload_file(Bucket = bucket_name, Filename = file_path, Key = destination_path)
|
|
|
|
def handle_zip(zip_path):
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
unzip(zip_path, tmp_dir)
|
|
print('unzipped')
|
|
local_shas = generate_local_shas(tmp_dir)
|
|
local_manifest = generate_manifest(local_shas)
|
|
|
|
remote_manifest = fetch_manifest()
|
|
|
|
remote_shas = read_shas(remote_manifest)
|
|
|
|
syncToS3(tmp_dir, local_shas, remote_shas)
|
|
|
|
s3_client = boto3.client('s3')
|
|
s3_client.put_object(Bucket = bucket_name, Body = local_manifest, Key = manifest_path)
|
|
|
|
def lambda_handler(event, context):
|
|
body = json.loads(event['body'])
|
|
artifacts = body['links']['artifacts']
|
|
files = artifacts[0]['files']
|
|
downloadLink = files[0]['href']
|
|
print("link: " + downloadLink)
|
|
|
|
with urllib.request.urlopen(downloadLink) as response:
|
|
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
|
|
shutil.copyfileobj(response, tmp_file)
|
|
print("copied to " + tmp_file.name)
|
|
|
|
handle_zip(tmp_file)
|
|
|
|
return {
|
|
'statusCode': 200,
|
|
'body': json.dumps('Success!')
|
|
}
|
|
|
|
|
|
# event = json.loads("""{
|
|
# "body": "{\\"links\\":{\\"artifacts\\":[{\\"files\\":[{\\"filename\\":\\"eagle0.zip\\",\\"href\\":\\"https://eagle0.net/assets/nolen777-new-unity-project-default-windows-desktop-64-bit-14.zip\\"}]}]}}\\n"
|
|
# }
|
|
# """)
|
|
|
|
# lambda_handler(event, None)
|