From 5002b7167041e04adeae85142816b157e8b6a6f8 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 19:38:50 +0200 Subject: [PATCH 01/14] use b2 upload-file instead of b2 sync --- .github/workflows/nightly.yml | 28 +++++++++++++++++----------- ci/upload-create-nightly.sh | 11 +++++++++++ 2 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 ci/upload-create-nightly.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1e1be0963..bb9a7784e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,7 +1,9 @@ name: Nightly + on: schedule: - cron: 0 20 * * * + push: jobs: build_windows: @@ -91,36 +93,40 @@ jobs: needs: [build_windows, build_macos, build_ubuntu] steps: - uses: actions/checkout@v1 + - name: Install B2 CLI shell: bash run: sudo pip install --upgrade b2 + - name: Download Windows artifacts uses: actions/download-artifact@v1 with: name: windows_artifacts + - name: Download Ubuntu artifacts uses: actions/download-artifact@v1 with: name: ubuntu_artifacts + - name: Download macOS artifacts uses: actions/download-artifact@v1 with: name: macos_artifacts - - name: Create archieves - run: | - now=$(date +'%Y-%m-%d') - 7z a output/odin-windows-amd64-nightly+$now.zip -r windows_artifacts/ - 7z a output/odin-ubuntu-amd64-nightly+$now.zip -r ubuntu_artifacts/ - 7z a output/odin-macos-amd64-nightly+$now.zip -r macos_artifact/ - - name: Upload artifacts to b2 + + - name: Create archives and upload shell: bash env: APPID: ${{ secrets.B2_APPID }} APPKEY: ${{ secrets.B2_APPKEY }} + BUCKET: ${{ secrets.B2_BUCKET }} run: | b2 authorize-account "$APPID" "$APPKEY" - b2 sync --keepDays 7 output b2://odin-binaries/nightly + + ./ci/upload-create-nightly.sh "$BUCKET" windows-amd64 windows_artifacts/ + ./ci/upload-create-nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ + ./ci/upload-create-nightly.sh "$BUCKET" macos-amd64 macos_artifact/ + python3 ci/create_nightly_json.py > nightly.json - b2 upload-file odin-binaries nightly.json nightly.json - b2 clear-account - + b2 upload-file "$BUCKET" nightly.json nightly.json + + b2 clear-account \ No newline at end of file diff --git a/ci/upload-create-nightly.sh b/ci/upload-create-nightly.sh new file mode 100644 index 000000000..0764cf00b --- /dev/null +++ b/ci/upload-create-nightly.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +bucket=$1 +platform=$2 +artifact=$2 + +now=$(date +'%Y-%m-%d') +filename="odin-$platform-nightly+$now.zip" + +7z a "output/$filename" -r "$artifact" +b2 upload-file "$bucket" "$filename" "$filename" \ No newline at end of file From 9170c875e12d9dc64da7abde3afa782fb84e8d69 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 19:44:26 +0200 Subject: [PATCH 02/14] chmod +x the script before execution --- .github/workflows/nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bb9a7784e..b82b6687e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -122,6 +122,7 @@ jobs: run: | b2 authorize-account "$APPID" "$APPKEY" + chmod +x ./ci/upload-create-nightly.sh ./ci/upload-create-nightly.sh "$BUCKET" windows-amd64 windows_artifacts/ ./ci/upload-create-nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ ./ci/upload-create-nightly.sh "$BUCKET" macos-amd64 macos_artifact/ From 802a7763308d6d52236117102bf2012b1fc74ef0 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 19:49:37 +0200 Subject: [PATCH 03/14] Use the correct path during upload --- ci/upload-create-nightly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/upload-create-nightly.sh b/ci/upload-create-nightly.sh index 0764cf00b..5f702f9c2 100644 --- a/ci/upload-create-nightly.sh +++ b/ci/upload-create-nightly.sh @@ -8,4 +8,4 @@ now=$(date +'%Y-%m-%d') filename="odin-$platform-nightly+$now.zip" 7z a "output/$filename" -r "$artifact" -b2 upload-file "$bucket" "$filename" "$filename" \ No newline at end of file +b2 upload-file --noProgress "$bucket" "output/$filename" "$filename" \ No newline at end of file From 1bd0e09ae130050be33f99afd3e53e43a069dbab Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 19:58:11 +0200 Subject: [PATCH 04/14] Upload to nightly path and echo out some info --- ci/upload-create-nightly.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/upload-create-nightly.sh b/ci/upload-create-nightly.sh index 5f702f9c2..763123646 100644 --- a/ci/upload-create-nightly.sh +++ b/ci/upload-create-nightly.sh @@ -7,5 +7,7 @@ artifact=$2 now=$(date +'%Y-%m-%d') filename="odin-$platform-nightly+$now.zip" -7z a "output/$filename" -r "$artifact" -b2 upload-file --noProgress "$bucket" "output/$filename" "$filename" \ No newline at end of file +echo "Creating archive $filename from $artifact and uploading to $bucket" + +7z a -bd "output/$filename" -r "$artifact" +b2 upload-file --noProgress "$bucket" "output/$filename" "nightly/$filename" \ No newline at end of file From 020856d91a12279c8d5ee3e3529b0c5af3884e6c Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 19:59:00 +0200 Subject: [PATCH 05/14] Fix macos artifact typo --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b82b6687e..58129effd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -125,7 +125,7 @@ jobs: chmod +x ./ci/upload-create-nightly.sh ./ci/upload-create-nightly.sh "$BUCKET" windows-amd64 windows_artifacts/ ./ci/upload-create-nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ - ./ci/upload-create-nightly.sh "$BUCKET" macos-amd64 macos_artifact/ + ./ci/upload-create-nightly.sh "$BUCKET" macos-amd64 macos_artifacts/ python3 ci/create_nightly_json.py > nightly.json b2 upload-file "$BUCKET" nightly.json nightly.json From 550df8711f8cd6d91b3c732999da71e3a3031ec0 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 20:04:43 +0200 Subject: [PATCH 06/14] Fix typo in upload-create-nightly.sh --- ci/upload-create-nightly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/upload-create-nightly.sh b/ci/upload-create-nightly.sh index 763123646..754b9b87c 100644 --- a/ci/upload-create-nightly.sh +++ b/ci/upload-create-nightly.sh @@ -2,7 +2,7 @@ bucket=$1 platform=$2 -artifact=$2 +artifact=$3 now=$(date +'%Y-%m-%d') filename="odin-$platform-nightly+$now.zip" From 2dcc986c4c829c505d2e3f70d2b7e289acfae5cf Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 20:11:00 +0200 Subject: [PATCH 07/14] Update /ci/build_ci.bat --- ci/build_ci.bat | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ci/build_ci.bat b/ci/build_ci.bat index 53da20680..5d5b30a74 100644 --- a/ci/build_ci.bat +++ b/ci/build_ci.bat @@ -1,8 +1,11 @@ @echo off +:: Make sure this is a decent name and not generic set exe_name=odin.exe -set compiler_flags= -nologo -Oi -TP -fp:precise -Gm- -MP -FC -GS- -EHsc- -GR- -O2 -MT -Z7 -DNO_ARRAY_BOUNDS_CHECK +set compiler_flags= -nologo -Oi -TP -fp:precise -Gm- -MP -FC -EHsc- -GR- -GF -O2 -MT -Z7 +set compiler_defines= -DLLVM_BACKEND_SUPPORT -DNO_ARRAY_BOUNDS_CHECK + set compiler_warnings= ^ -W4 -WX ^ -wd4100 -wd4101 -wd4127 -wd4189 ^ @@ -12,13 +15,17 @@ set compiler_warnings= ^ set compiler_includes= set libs= ^ - kernel32.lib + kernel32.lib ^ + bin\llvm\windows\LLVM-C.lib set linker_flags= -incremental:no -opt:ref -subsystem:console -debug -set compiler_settings=%compiler_includes% %compiler_flags% %compiler_warnings% +set compiler_settings=%compiler_includes% %compiler_flags% %compiler_warnings% %compiler_defines% set linker_settings=%libs% %linker_flags% -cl %compiler_settings% "src\main.cpp" ^ - /link %linker_settings% -OUT:%exe_name% ^ +del *.pdb > NUL 2> NUL +del *.ilk > NUL 2> NUL +cl %compiler_settings% "src\main.cpp" /link %linker_settings% -OUT:%exe_name% + +:end_of_build From df5626cc1f67112840e05ee8f2b9bdf8fa1f3a3b Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 20:13:44 +0200 Subject: [PATCH 08/14] Pass the bucket to /ci/create_nightly_json.py --- .github/workflows/nightly.yml | 2 +- ci/create_nightly_json.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 58129effd..a5dadb7fe 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -127,7 +127,7 @@ jobs: ./ci/upload-create-nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ ./ci/upload-create-nightly.sh "$BUCKET" macos-amd64 macos_artifacts/ - python3 ci/create_nightly_json.py > nightly.json + python3 ci/create_nightly_json.py "$BUCKET" > nightly.json b2 upload-file "$BUCKET" nightly.json nightly.json b2 clear-account \ No newline at end of file diff --git a/ci/create_nightly_json.py b/ci/create_nightly_json.py index 43193a65d..65be4f7af 100644 --- a/ci/create_nightly_json.py +++ b/ci/create_nightly_json.py @@ -3,18 +3,20 @@ import sys import json import datetime import urllib.parse +import sys def main(): files_by_date = {} + bucket = sys.argv[1] - files_lines = execute_cli("b2 ls --long odin-binaries nightly").split("\n") + files_lines = execute_cli(f"b2 ls --long {bucket} nightly").split("\n") for x in files_lines: parts = x.split(" ", 1) if parts[0]: json_str = execute_cli(f"b2 get-file-info {parts[0]}") data = json.loads(json_str) name = remove_prefix(data['fileName'], "nightly/") - url = f"https://f001.backblazeb2.com/file/odin-binaries/nightly/{urllib.parse.quote_plus(name)}" + url = f"https://f001.backblazeb2.com/file/{bucket}/nightly/{urllib.parse.quote_plus(name)}" sha1 = data['contentSha1'] ts = int(data['fileInfo']['src_last_modified_millis']) date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d') From d0cee153172dba62c6f74000196e16c6ae41855b Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Wed, 15 Apr 2020 20:44:47 +0200 Subject: [PATCH 09/14] Remove on:push in nightly.yml --- .github/workflows/nightly.yml | 7 +++---- ci/{upload-create-nightly.sh => upload_create_nightly.sh} | 0 2 files changed, 3 insertions(+), 4 deletions(-) rename ci/{upload-create-nightly.sh => upload_create_nightly.sh} (100%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a5dadb7fe..11b0aa9a4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,7 +3,6 @@ name: Nightly on: schedule: - cron: 0 20 * * * - push: jobs: build_windows: @@ -123,9 +122,9 @@ jobs: b2 authorize-account "$APPID" "$APPKEY" chmod +x ./ci/upload-create-nightly.sh - ./ci/upload-create-nightly.sh "$BUCKET" windows-amd64 windows_artifacts/ - ./ci/upload-create-nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ - ./ci/upload-create-nightly.sh "$BUCKET" macos-amd64 macos_artifacts/ + ./ci/upload_create_nightly.sh "$BUCKET" windows-amd64 windows_artifacts/ + ./ci/upload_create_nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ + ./ci/upload_create_nightly.sh "$BUCKET" macos-amd64 macos_artifacts/ python3 ci/create_nightly_json.py "$BUCKET" > nightly.json b2 upload-file "$BUCKET" nightly.json nightly.json diff --git a/ci/upload-create-nightly.sh b/ci/upload_create_nightly.sh similarity index 100% rename from ci/upload-create-nightly.sh rename to ci/upload_create_nightly.sh From a55975bd5a5dd1ff4ffbae083224dc8e15f06235 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sun, 19 Apr 2020 18:00:54 +0200 Subject: [PATCH 10/14] Testing --- .github/workflows/nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 11b0aa9a4..671b368b4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,6 +3,7 @@ name: Nightly on: schedule: - cron: 0 20 * * * + push: jobs: build_windows: From 7fddac2c364d31d6b93efc188b94255a8f96c4a7 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sun, 19 Apr 2020 18:06:20 +0200 Subject: [PATCH 11/14] Fix typo in nightly.yml --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 671b368b4..3e3c97011 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -122,7 +122,7 @@ jobs: run: | b2 authorize-account "$APPID" "$APPKEY" - chmod +x ./ci/upload-create-nightly.sh + chmod +x ./ci/upload_create_nightly.sh ./ci/upload_create_nightly.sh "$BUCKET" windows-amd64 windows_artifacts/ ./ci/upload_create_nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ ./ci/upload_create_nightly.sh "$BUCKET" macos-amd64 macos_artifacts/ From 3dac1c34fa963d6a4828f36a2b6f1d6fc3722d8b Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sun, 19 Apr 2020 21:34:04 +0200 Subject: [PATCH 12/14] Script to delete files older than x days --- .github/workflows/nightly.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3e3c97011..ac4fcb74e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -119,6 +119,7 @@ jobs: APPID: ${{ secrets.B2_APPID }} APPKEY: ${{ secrets.B2_APPKEY }} BUCKET: ${{ secrets.B2_BUCKET }} + DAYS_TO_KEEP: ${{ secrets.B2_DAYS_TO_KEEP }} run: | b2 authorize-account "$APPID" "$APPKEY" @@ -127,6 +128,8 @@ jobs: ./ci/upload_create_nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/ ./ci/upload_create_nightly.sh "$BUCKET" macos-amd64 macos_artifacts/ + python3 ci/delete_old_binaries.py "$BUCKET" "$DAYS_TO_KEEP" + python3 ci/create_nightly_json.py "$BUCKET" > nightly.json b2 upload-file "$BUCKET" nightly.json nightly.json From 8eda24f2d180a89905f45f8ae4bf6c29e0c18ae6 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sun, 19 Apr 2020 21:38:59 +0200 Subject: [PATCH 13/14] Actually include script to delete binaries --- ci/delete_old_binaries.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 ci/delete_old_binaries.py diff --git a/ci/delete_old_binaries.py b/ci/delete_old_binaries.py new file mode 100644 index 000000000..206d849f5 --- /dev/null +++ b/ci/delete_old_binaries.py @@ -0,0 +1,34 @@ +import subprocess +import sys +import json +import datetime +import urllib.parse +import sys + +def main(): + files_by_date = {} + bucket = sys.argv[1] + days_to_keep = int(sys.argv[2]) + print(f"Looking for binaries to delete older than {days_to_keep} days") + + files_lines = execute_cli(f"b2 ls --long --versions {bucket} nightly").split("\n") + for x in files_lines: + parts = [y for y in x.split(' ') if y] + + if parts and parts[0]: + date = datetime.datetime.strptime(parts[2], '%Y-%m-%d').replace(hour=0, minute=0, second=0, microsecond=0) + now = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + delta = now - date + + if delta.days > days_to_keep: + print(f'Deleting {parts[5]}') + execute_cli(f'b2 delete-file-version {parts[0]}') + + +def execute_cli(command): + sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + return sb.stdout.read().decode("utf-8"); + +if __name__ == '__main__': + sys.exit(main()) + From 680d723c779a98eb092b4f16f043e9f8e4e1e6b2 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sun, 19 Apr 2020 21:44:45 +0200 Subject: [PATCH 14/14] Remove push --- .github/workflows/nightly.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ac4fcb74e..c6b96b583 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,7 +3,6 @@ name: Nightly on: schedule: - cron: 0 20 * * * - push: jobs: build_windows: