diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1e1be0963..c6b96b583 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,4 +1,5 @@ name: Nightly + on: schedule: - cron: 0 20 * * * @@ -91,36 +92,44 @@ jobs: needs: [build_windows, build_macos, build_ubuntu] steps: - uses: actions/checkout@v1 + - name: Install B2 CLI shell: bash run: sudo pip install --upgrade b2 + - name: Download Windows artifacts uses: actions/download-artifact@v1 with: name: windows_artifacts + - name: Download Ubuntu artifacts uses: actions/download-artifact@v1 with: name: ubuntu_artifacts + - name: Download macOS artifacts uses: actions/download-artifact@v1 with: name: macos_artifacts - - name: Create archieves - run: | - now=$(date +'%Y-%m-%d') - 7z a output/odin-windows-amd64-nightly+$now.zip -r windows_artifacts/ - 7z a output/odin-ubuntu-amd64-nightly+$now.zip -r ubuntu_artifacts/ - 7z a output/odin-macos-amd64-nightly+$now.zip -r macos_artifact/ - - name: Upload artifacts to b2 + + - name: Create archives and upload shell: bash env: APPID: ${{ secrets.B2_APPID }} APPKEY: ${{ secrets.B2_APPKEY }} + BUCKET: ${{ secrets.B2_BUCKET }} + DAYS_TO_KEEP: ${{ secrets.B2_DAYS_TO_KEEP }} run: | b2 authorize-account "$APPID" "$APPKEY" - b2 sync --keepDays 7 output b2://odin-binaries/nightly - python3 ci/create_nightly_json.py > nightly.json - b2 upload-file odin-binaries nightly.json nightly.json - b2 clear-account - + + chmod +x ./ci/upload_create_nightly.sh + ./ci/upload_create_nightly.sh "$BUCKET" windows-amd64 windows_artifacts/ + ./ci/upload_create_nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ + ./ci/upload_create_nightly.sh "$BUCKET" macos-amd64 macos_artifacts/ + + python3 ci/delete_old_binaries.py "$BUCKET" "$DAYS_TO_KEEP" + + python3 ci/create_nightly_json.py "$BUCKET" > nightly.json + b2 upload-file "$BUCKET" nightly.json nightly.json + + b2 clear-account \ No newline at end of file diff --git a/ci/build_ci.bat b/ci/build_ci.bat index 53da20680..5d5b30a74 100644 --- a/ci/build_ci.bat +++ b/ci/build_ci.bat @@ -1,8 +1,11 @@ @echo off +:: Make sure this is a decent name and not generic set exe_name=odin.exe -set compiler_flags= -nologo -Oi -TP -fp:precise -Gm- -MP -FC -GS- -EHsc- -GR- -O2 -MT -Z7 -DNO_ARRAY_BOUNDS_CHECK +set compiler_flags= -nologo -Oi -TP -fp:precise -Gm- -MP -FC -EHsc- -GR- -GF -O2 -MT -Z7 +set compiler_defines= -DLLVM_BACKEND_SUPPORT -DNO_ARRAY_BOUNDS_CHECK + set compiler_warnings= ^ -W4 -WX ^ -wd4100 -wd4101 -wd4127 -wd4189 ^ @@ -12,13 +15,17 @@ set compiler_warnings= ^ set compiler_includes= set libs= ^ - kernel32.lib + kernel32.lib ^ + bin\llvm\windows\LLVM-C.lib set linker_flags= -incremental:no -opt:ref -subsystem:console -debug -set compiler_settings=%compiler_includes% %compiler_flags% %compiler_warnings% +set compiler_settings=%compiler_includes% %compiler_flags% %compiler_warnings% %compiler_defines% set linker_settings=%libs% %linker_flags% -cl %compiler_settings% "src\main.cpp" ^ - /link %linker_settings% -OUT:%exe_name% ^ +del *.pdb > NUL 2> NUL +del *.ilk > NUL 2> NUL +cl %compiler_settings% "src\main.cpp" /link %linker_settings% -OUT:%exe_name% + +:end_of_build diff --git a/ci/create_nightly_json.py b/ci/create_nightly_json.py index 43193a65d..65be4f7af 100644 --- a/ci/create_nightly_json.py +++ b/ci/create_nightly_json.py @@ -3,18 +3,20 @@ import sys import json import datetime import urllib.parse +import sys def main(): files_by_date = {} + bucket = sys.argv[1] - files_lines = execute_cli("b2 ls --long odin-binaries nightly").split("\n") + files_lines = execute_cli(f"b2 ls --long {bucket} nightly").split("\n") for x in files_lines: parts = x.split(" ", 1) if parts[0]: json_str = execute_cli(f"b2 get-file-info {parts[0]}") data = json.loads(json_str) name = remove_prefix(data['fileName'], "nightly/") - url = f"https://f001.backblazeb2.com/file/odin-binaries/nightly/{urllib.parse.quote_plus(name)}" + url = f"https://f001.backblazeb2.com/file/{bucket}/nightly/{urllib.parse.quote_plus(name)}" sha1 = data['contentSha1'] ts = int(data['fileInfo']['src_last_modified_millis']) date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d') diff --git a/ci/delete_old_binaries.py b/ci/delete_old_binaries.py new file mode 100644 index 000000000..206d849f5 --- /dev/null +++ b/ci/delete_old_binaries.py @@ -0,0 +1,34 @@ +import subprocess +import sys +import json +import datetime +import urllib.parse +import sys + +def main(): + files_by_date = {} + bucket = sys.argv[1] + days_to_keep = int(sys.argv[2]) + print(f"Looking for binaries to delete older than {days_to_keep} days") + + files_lines = execute_cli(f"b2 ls --long --versions {bucket} nightly").split("\n") + for x in files_lines: + parts = [y for y in x.split(' ') if y] + + if parts and parts[0]: + date = datetime.datetime.strptime(parts[2], '%Y-%m-%d').replace(hour=0, minute=0, second=0, microsecond=0) + now = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + delta = now - date + + if delta.days > days_to_keep: + print(f'Deleting {parts[5]}') + execute_cli(f'b2 delete-file-version {parts[0]}') + + +def execute_cli(command): + sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + return sb.stdout.read().decode("utf-8"); + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/ci/upload_create_nightly.sh b/ci/upload_create_nightly.sh new file mode 100644 index 000000000..754b9b87c --- /dev/null +++ b/ci/upload_create_nightly.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +bucket=$1 +platform=$2 +artifact=$3 + +now=$(date +'%Y-%m-%d') +filename="odin-$platform-nightly+$now.zip" + +echo "Creating archive $filename from $artifact and uploading to $bucket" + +7z a -bd "output/$filename" -r "$artifact" +b2 upload-file --noProgress "$bucket" "output/$filename" "nightly/$filename" \ No newline at end of file