mirror of
https://github.com/Ed94/Odin.git
synced 2026-08-05 15:18:49 +00:00
Rewrite upload_b2 nightly action against B2 SDK
This commit is contained in:
Jeroen van Rijn
@@ -1,51 +0,0 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import json
|
||||
import datetime
|
||||
import urllib.parse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
files_by_date = {}
|
||||
bucket = sys.argv[1]
|
||||
|
||||
files_lines = execute_cli(f"b2 ls --long b2://{bucket}/nightly/").split("\n")
|
||||
for x in files_lines:
|
||||
parts = x.split(" ", 1)
|
||||
if parts[0]:
|
||||
print(f"Parts[0]: {parts[0]}", flush=True)
|
||||
json_str = execute_cli(f"b2 file info b2://{bucket}/{parts[0]}")
|
||||
data = json.loads(json_str)
|
||||
name = remove_prefix(data['fileName'], "nightly/")
|
||||
url = f"https://f001.backblazeb2.com/file/{bucket}/nightly/{urllib.parse.quote_plus(name)}"
|
||||
sha1 = data['contentSha1']
|
||||
size = int(data['size'])
|
||||
ts = int(data['fileInfo']['src_last_modified_millis'])
|
||||
date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d')
|
||||
|
||||
if date not in files_by_date.keys():
|
||||
files_by_date[date] = []
|
||||
|
||||
files_by_date[date].append({
|
||||
'name': name,
|
||||
'url': url,
|
||||
'sha1': sha1,
|
||||
'sizeInBytes': size,
|
||||
})
|
||||
|
||||
now = datetime.datetime.utcnow().isoformat()
|
||||
|
||||
print(json.dumps({
|
||||
'last_updated' : now,
|
||||
'files': files_by_date
|
||||
}, sort_keys=True, indent=4))
|
||||
|
||||
def remove_prefix(text, prefix):
|
||||
return text[text.startswith(prefix) and len(prefix):]
|
||||
|
||||
def execute_cli(command):
|
||||
sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
|
||||
return sb.stdout.read().decode("utf-8");
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,33 +0,0 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import json
|
||||
import datetime
|
||||
import urllib.parse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
files_by_date = {}
|
||||
bucket = sys.argv[1]
|
||||
days_to_keep = int(sys.argv[2])
|
||||
print(f"Looking for binaries to delete older than {days_to_keep} days")
|
||||
|
||||
files_lines = execute_cli(f"b2 ls --long --versions b2://{bucket}/nightly/").split("\n")
|
||||
for x in files_lines:
|
||||
parts = [y for y in x.split(' ') if y]
|
||||
|
||||
if parts and parts[0]:
|
||||
date = datetime.datetime.strptime(parts[2], '%Y-%m-%d').replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
now = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
delta = now - date
|
||||
|
||||
if delta.days > days_to_keep:
|
||||
print(f'Deleting b2://{bucket}/{parts[5]}')
|
||||
execute_cli(f'b2 rm b2://{bucket}/{parts[5]}')
|
||||
|
||||
|
||||
def execute_cli(command):
|
||||
sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
|
||||
return sb.stdout.read().decode("utf-8");
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
import os
|
||||
import sys
|
||||
from zipfile import ZipFile
|
||||
from b2sdk.v2 import InMemoryAccountInfo, B2Api
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
UPLOAD_FOLDER = "nightly/"
|
||||
|
||||
info = InMemoryAccountInfo()
|
||||
b2_api = B2Api(info)
|
||||
application_key_id = os.environ['APPID']
|
||||
application_key = os.environ['APPKEY']
|
||||
bucket_name = os.environ['BUCKET']
|
||||
days_to_keep = os.environ['DAYS_TO_KEEP']
|
||||
|
||||
def auth() -> bool:
|
||||
try:
|
||||
realm = b2_api.account_info.get_realm()
|
||||
return True # Already authenticated
|
||||
except:
|
||||
pass # Not yet authenticated
|
||||
|
||||
err = b2_api.authorize_account("production", application_key_id, application_key)
|
||||
return err == None
|
||||
|
||||
def get_bucket():
|
||||
if not auth(): sys.exit(1)
|
||||
return b2_api.get_bucket_by_name(bucket_name)
|
||||
|
||||
def remove_prefix(text: str, prefix: str) -> str:
|
||||
return text[text.startswith(prefix) and len(prefix):]
|
||||
|
||||
def create_and_upload_artifact_zip(platform: str, artifact: str) -> int:
|
||||
now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
destination_zip_name = "odin-{}-nightly+{}.zip".format(platform, now.strftime("%Y-%m-%d"))
|
||||
|
||||
source_zip_name = artifact
|
||||
if not artifact.endswith(".zip"):
|
||||
print(f"Creating archive {destination_zip_name} from {artifact} and uploading to {bucket_name}")
|
||||
|
||||
source_zip_name = destination_zip_name
|
||||
with ZipFile(source_zip_name, 'w') as z:
|
||||
for root, directory, filenames in os.walk(artifact):
|
||||
for file in filenames:
|
||||
file_path = os.path.join(root, file)
|
||||
zip_path = os.path.join("dist", os.path.relpath(file_path, artifact))
|
||||
z.write(file_path, zip_path)
|
||||
|
||||
if not os.path.exists(source_zip_name):
|
||||
print(f"Error: Newly created ZIP archive {source_zip_name} not found.")
|
||||
return 1
|
||||
|
||||
print("Uploading {} to {}".format(source_zip_name, UPLOAD_FOLDER + destination_zip_name))
|
||||
bucket = get_bucket()
|
||||
res = bucket.upload_local_file(
|
||||
source_zip_name, # Local file to upload
|
||||
"nightly/" + destination_zip_name, # B2 destination path
|
||||
)
|
||||
return 0
|
||||
|
||||
def prune_artifacts():
|
||||
print(f"Looking for binaries to delete older than {days_to_keep} days")
|
||||
|
||||
bucket = get_bucket()
|
||||
for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=False):
|
||||
# Timestamp is in milliseconds
|
||||
date = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
delta = now - date
|
||||
|
||||
if delta.days > int(days_to_keep):
|
||||
print("Deleting {}".format(file.file_name))
|
||||
file.delete()
|
||||
|
||||
return 0
|
||||
|
||||
def update_nightly_json():
|
||||
print(f"Updating nightly.json with files {days_to_keep} days or newer")
|
||||
|
||||
files_by_date = {}
|
||||
|
||||
bucket = get_bucket()
|
||||
|
||||
for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=True):
|
||||
# Timestamp is in milliseconds
|
||||
date = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0).strftime('%Y-%m-%d')
|
||||
name = remove_prefix(file.file_name, UPLOAD_FOLDER)
|
||||
sha1 = file.content_sha1
|
||||
size = file.size
|
||||
url = bucket.get_download_url(file.file_name)
|
||||
|
||||
if date not in files_by_date.keys():
|
||||
files_by_date[date] = []
|
||||
|
||||
files_by_date[date].append({
|
||||
'name': name,
|
||||
'url': url,
|
||||
'sha1': sha1,
|
||||
'sizeInBytes': size,
|
||||
})
|
||||
|
||||
now = datetime.utcnow().isoformat()
|
||||
|
||||
nightly = json.dumps({
|
||||
'last_updated' : now,
|
||||
'files': files_by_date
|
||||
}, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8')
|
||||
|
||||
res = bucket.upload_bytes(
|
||||
nightly, # JSON bytes
|
||||
"nightly.json", # B2 destination path
|
||||
)
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 1:
|
||||
print("Usage: {} <verb> [arguments]".format(sys.argv[0]))
|
||||
print("\tartifact <platform prefix> <artifact path>\n\t\tCreates and uploads a platform artifact zip.")
|
||||
print("\tprune\n\t\tDeletes old artifacts from bucket")
|
||||
print("\tjson\n\t\tUpdate and upload nightly.json")
|
||||
sys.exit(1)
|
||||
else:
|
||||
command = sys.argv[1].lower()
|
||||
if command == "artifact":
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: {} artifact <platform prefix> <artifact path>".format(sys.argv[0]))
|
||||
print("Error: Expected artifact command to be given platform prefix and artifact path.\n")
|
||||
sys.exit(1)
|
||||
|
||||
res = create_and_upload_artifact_zip(sys.argv[2], sys.argv[3])
|
||||
sys.exit(res)
|
||||
|
||||
elif command == "prune":
|
||||
res = prune_artifacts()
|
||||
sys.exit(res)
|
||||
|
||||
elif command == "json":
|
||||
res = update_nightly_json()
|
||||
sys.exit(res)
|
||||
@@ -1,25 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
bucket=$1
|
||||
platform=$2
|
||||
artifact=$3
|
||||
|
||||
now=$(date +'%Y-%m-%d')
|
||||
filename="odin-$platform-nightly+$now.zip"
|
||||
|
||||
echo "Creating archive $filename from $artifact and uploading to $bucket"
|
||||
|
||||
# If this is already zipped up (done before artifact upload to keep permissions in tact), just move it.
|
||||
if [ "${artifact: -4}" == ".zip" ]
|
||||
then
|
||||
echo "Artifact already a zip"
|
||||
mkdir -p "output"
|
||||
mv "$artifact" "output/$filename"
|
||||
else
|
||||
echo "Artifact needs to be zipped"
|
||||
7z a -bd "output/$filename" -r "$artifact"
|
||||
fi
|
||||
|
||||
b2 file upload "$bucket" "output/$filename" "nightly/$filename"
|
||||
Reference in New Issue
Block a user