Merge pull request #607 from ThisDrunkDane/nightly-build

Use b2 upload-file instead of b2 sync for nightly builds

.github/workflows/nightly.yml

@@ -1,4 +1,5 @@
 name:  Nightly
+
 on:
   schedule:
     - cron: 0 20 * * *
@@ -91,36 +92,44 @@ jobs:
     needs: [build_windows, build_macos, build_ubuntu]
     steps:
       - uses: actions/checkout@v1
+      
       - name: Install B2 CLI
         shell: bash
         run: sudo pip install --upgrade b2
+      
       - name: Download Windows artifacts
         uses: actions/download-artifact@v1
         with:
           name: windows_artifacts
+      
       - name: Download Ubuntu artifacts
         uses: actions/download-artifact@v1
         with:
           name: ubuntu_artifacts
+      
       - name: Download macOS artifacts
         uses: actions/download-artifact@v1
         with:
           name: macos_artifacts
-      - name: Create archieves
+
-        run: |
+      - name: Create archives and upload
-          now=$(date +'%Y-%m-%d')
-          7z a output/odin-windows-amd64-nightly+$now.zip -r windows_artifacts/
-          7z a output/odin-ubuntu-amd64-nightly+$now.zip -r ubuntu_artifacts/
-          7z a output/odin-macos-amd64-nightly+$now.zip -r macos_artifact/
-      - name: Upload artifacts to b2
         shell: bash
         env:
           APPID: ${{ secrets.B2_APPID }}
           APPKEY: ${{ secrets.B2_APPKEY }}
+          BUCKET: ${{ secrets.B2_BUCKET }}
+          DAYS_TO_KEEP: ${{ secrets.B2_DAYS_TO_KEEP }}
         run: |
           b2 authorize-account "$APPID" "$APPKEY"
-          b2 sync --keepDays 7 output b2://odin-binaries/nightly
+
-          python3 ci/create_nightly_json.py > nightly.json
+          chmod +x ./ci/upload_create_nightly.sh
-          b2 upload-file odin-binaries nightly.json nightly.json
+          ./ci/upload_create_nightly.sh "$BUCKET" windows-amd64 windows_artifacts/
-          b2 clear-account
+          ./ci/upload_create_nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/
-        
+          ./ci/upload_create_nightly.sh "$BUCKET" macos-amd64 macos_artifacts/
+
+          python3 ci/delete_old_binaries.py "$BUCKET" "$DAYS_TO_KEEP"
+
+          python3 ci/create_nightly_json.py "$BUCKET" > nightly.json
+          b2 upload-file "$BUCKET" nightly.json nightly.json
+
+          b2 clear-account

ci/build_ci.bat

@@ -1,8 +1,11 @@
 @echo off
 
+:: Make sure this is a decent name and not generic
 set exe_name=odin.exe
 
-set compiler_flags= -nologo -Oi -TP -fp:precise -Gm- -MP -FC -GS- -EHsc- -GR- -O2 -MT -Z7 -DNO_ARRAY_BOUNDS_CHECK
+set compiler_flags= -nologo -Oi -TP -fp:precise -Gm- -MP -FC -EHsc- -GR- -GF -O2 -MT -Z7
+set compiler_defines= -DLLVM_BACKEND_SUPPORT -DNO_ARRAY_BOUNDS_CHECK
+
 set compiler_warnings= ^
     -W4 -WX ^
     -wd4100 -wd4101 -wd4127 -wd4189 ^
@@ -12,13 +15,17 @@ set compiler_warnings= ^
 
 set compiler_includes=
 set libs= ^
-    kernel32.lib
+    kernel32.lib ^
+    bin\llvm\windows\LLVM-C.lib
 
 set linker_flags= -incremental:no -opt:ref -subsystem:console -debug
 
-set compiler_settings=%compiler_includes% %compiler_flags% %compiler_warnings%
+set compiler_settings=%compiler_includes% %compiler_flags% %compiler_warnings% %compiler_defines%
 set linker_settings=%libs% %linker_flags%
 
-cl %compiler_settings% "src\main.cpp" ^
+del *.pdb > NUL 2> NUL
-    /link %linker_settings% -OUT:%exe_name% ^
+del *.ilk > NUL 2> NUL
 
+cl %compiler_settings% "src\main.cpp" /link %linker_settings% -OUT:%exe_name%
+    
+:end_of_build

ci/create_nightly_json.py

@@ -3,18 +3,20 @@ import sys
 import json
 import datetime
 import urllib.parse
+import sys
 
 def main():
     files_by_date = {}
+    bucket = sys.argv[1]
 
-    files_lines = execute_cli("b2 ls --long odin-binaries nightly").split("\n")
+    files_lines = execute_cli(f"b2 ls --long {bucket} nightly").split("\n")
     for x in files_lines:
         parts = x.split(" ", 1)
         if parts[0]:
             json_str = execute_cli(f"b2 get-file-info {parts[0]}")
             data = json.loads(json_str)
             name = remove_prefix(data['fileName'], "nightly/")
-            url = f"https://f001.backblazeb2.com/file/odin-binaries/nightly/{urllib.parse.quote_plus(name)}"
+            url = f"https://f001.backblazeb2.com/file/{bucket}/nightly/{urllib.parse.quote_plus(name)}"
             sha1 = data['contentSha1']
             ts = int(data['fileInfo']['src_last_modified_millis'])
             date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d')

ci/delete_old_binaries.py

@@ -0,0 +1,34 @@
+import subprocess
+import sys
+import json
+import datetime
+import urllib.parse
+import sys
+
+def main():
+    files_by_date = {}
+    bucket = sys.argv[1]
+    days_to_keep = int(sys.argv[2])
+    print(f"Looking for binaries to delete older than {days_to_keep} days")
+
+    files_lines = execute_cli(f"b2 ls --long --versions {bucket} nightly").split("\n")
+    for x in files_lines:
+        parts = [y for y in x.split(' ') if y]
+
+        if parts and parts[0]:
+            date = datetime.datetime.strptime(parts[2], '%Y-%m-%d').replace(hour=0, minute=0, second=0, microsecond=0)
+            now = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
+            delta = now - date
+
+            if delta.days > days_to_keep:
+                print(f'Deleting {parts[5]}')
+                execute_cli(f'b2 delete-file-version {parts[0]}')
+
+
+def execute_cli(command):
+    sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+    return sb.stdout.read().decode("utf-8");
+
+if __name__ == '__main__':
+    sys.exit(main())
+

ci/upload_create_nightly.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+bucket=$1
+platform=$2
+artifact=$3
+
+now=$(date +'%Y-%m-%d')
+filename="odin-$platform-nightly+$now.zip"
+
+echo "Creating archive $filename from $artifact and uploading to $bucket"
+
+7z a -bd "output/$filename" -r "$artifact"
+b2 upload-file --noProgress "$bucket" "output/$filename" "nightly/$filename"