From 2df0532b179a4d435d73126c6caba345d2131705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20Hjortsh=C3=B8j?= Date: Sat, 11 Apr 2020 17:32:21 +0200 Subject: [PATCH 1/7] Create nightly.yml --- .github/workflows/nightly.yml | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..c682ca454 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,48 @@ +name: Nightly +on: [push] + +jobs: + build: + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + + - name: Install cURL + run: choco install curl + - name: Download and unpack LLVM bins + shell: cmd + run: | + cd bin + curl -sL https://github.com/odin-lang/Odin/releases/download/llvm-windows/llvm-binaries.zip --output llvm-binaries.zip + 7z x llvm-binaries.zip > nul + - name: build Odin + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat + ./build_ci.bat + - name: Odin run + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat + odin run examples/demo/demo.odin + - name: upload artifact + uses: actions/upload-artifact@v1 + with: + name: exe + path: odin.exe + upload_b2: + runs-on: [linux] + needs: [build] + container: + image: python:3.7-alpine + steps: + - name: Install B2 CLI + shell: bash + run: sudo pip install --upgrade b2 + - name: download exe + uses: actions/download-artifact@v1 + with: + name: exe + - name: List artifact + run: find . -type f -printf "%T@ %p\n" | sort -nr | cut -d\ -f2- + From df80e8752b39e45b3466330625e5db5dc4925aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20Hjortsh=C3=B8j?= Date: Sat, 11 Apr 2020 17:36:30 +0200 Subject: [PATCH 2/7] Update nightly.yml --- .github/workflows/nightly.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c682ca454..24804b9f8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,7 +25,7 @@ jobs: run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat odin run examples/demo/demo.odin - - name: upload artifact + - name: Upload artifact uses: actions/upload-artifact@v1 with: name: exe @@ -33,16 +33,14 @@ jobs: upload_b2: runs-on: [linux] needs: [build] - container: - image: python:3.7-alpine steps: - name: Install B2 CLI shell: bash run: sudo pip install --upgrade b2 - - name: download exe + - name: Download exe uses: actions/download-artifact@v1 with: name: exe - - name: List artifact + - name: List artifacts run: find . -type f -printf "%T@ %p\n" | sort -nr | cut -d\ -f2- From c9bec10a8ee3c11b38a41822c3c573786c0f5e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20Hjortsh=C3=B8j?= Date: Sat, 11 Apr 2020 17:46:44 +0200 Subject: [PATCH 3/7] Setup POC nightly.yml --- .github/workflows/nightly.yml | 93 ++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 24804b9f8..7d67d370d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,11 +2,10 @@ name: Nightly on: [push] jobs: - build: + build_windows: runs-on: windows-latest steps: - uses: actions/checkout@v1 - - name: Install cURL run: choco install curl - name: Download and unpack LLVM bins @@ -25,22 +24,100 @@ jobs: run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat odin run examples/demo/demo.odin + - name: Copy artifacts + run: | + mkdir dist + cp odin.exe dist + cp -r shared dist + cp -r core dist + cp -r bin dist + cp -r examples dist - name: Upload artifact uses: actions/upload-artifact@v1 with: - name: exe - path: odin.exe + name: windows_artifacts + path: dist + build_ubuntu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: (Linux) Download LLVM + run: sudo apt-get install llvm + - name: build odin + run: make release + - name: Odin run + run: ./odin run examples/demo/demo.odin + - name: Copy artifacts + run: | + mkdir dist + cp odin dist + cp -r shared dist + cp -r core dist + cp -r examples dist + - name: Upload artifact + uses: actions/upload-artifact@v1 + with: + name: ubuntu_artifacts + path: dist + build_macos: + runs-on: macOS-latest + steps: + - uses: actions/checkout@v1 + - name: Download LLVM and setup PATH + run: | + brew install llvm + echo ::add-path::/usr/local/opt/llvm/bin + echo ::set-env name=CPATH::`xcrun --show-sdk-path`/usr/include + - name: build odin + run: make release + - name: Odin run + run: ./odin run examples/demo/demo.odin + - name: Copy artifacts + run: | + mkdir dist + cp odin dist + cp -r shared dist + cp -r core dist + cp -r examples dist + - name: Upload artifact + uses: actions/upload-artifact@v1 + with: + name: macos_artifacts + path: dist upload_b2: - runs-on: [linux] - needs: [build] + runs-on: [ubuntu-latest] + needs: [build_windows, build_macos, build_ubuntu] steps: - name: Install B2 CLI shell: bash run: sudo pip install --upgrade b2 - - name: Download exe + - name: Download Windows artifacts uses: actions/download-artifact@v1 with: - name: exe + name: windows_artifacts + - name: Download Ubuntu artifacts + uses: actions/download-artifact@v1 + with: + name: ubuntu_artifacts + - name: Download macOS artifacts + uses: actions/download-artifact@v1 + with: + name: macos_artifacts - name: List artifacts run: find . -type f -printf "%T@ %p\n" | sort -nr | cut -d\ -f2- + - name: Create archieves + run: | + now=$(date +'%Y-%m-%d') + 7z a test/odin-windows-amd64-nightly+$now.zip windows_artifacts/* + 7z a test/odin-ubuntu-amd64-nightly+$now.zip ubuntu_artifacts/* + 7z a test/odin-macos-amd64-nightly+$now.zip macos_artifacts/* + - name: Upload artifacts to b2 + shell: bash + env: + APPID: ${{ secrets.B2_APPID }} + APPKEY: ${{ secrets.B2_APPKEY }} + run: | + b2 authorize-account "$APPID" "$APPKEY" + b2 sync --delete test b2://odin-binaries/nightly + b2 clear-account From a615402d7c079c188a7e3fa3f4f59428aa52511c Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sat, 11 Apr 2020 19:55:20 +0200 Subject: [PATCH 4/7] Start on json script --- .github/workflows/ci.yml | 2 +- build_ci.bat => ci/build_ci.bat | 0 ci/create_nightly_json.sh | 5 +++++ 3 files changed, 6 insertions(+), 1 deletion(-) rename build_ci.bat => ci/build_ci.bat (100%) create mode 100644 ci/create_nightly_json.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c43b01e1..504dc1fec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat - ./build_ci.bat + ./ci/build_ci.bat - name: Odin run shell: cmd run: | diff --git a/build_ci.bat b/ci/build_ci.bat similarity index 100% rename from build_ci.bat rename to ci/build_ci.bat diff --git a/ci/create_nightly_json.sh b/ci/create_nightly_json.sh new file mode 100644 index 000000000..184423195 --- /dev/null +++ b/ci/create_nightly_json.sh @@ -0,0 +1,5 @@ +FILE_IDS=$(b2 ls --long odin-binaries nightly | cut -d ' ' -f 1) + +while IFS= read -r line; do + echo "... $line ..." +done <<< "$list" \ No newline at end of file From 9b9a4fcf22929e2008f54b4cbd03dc04045c4eac Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sat, 11 Apr 2020 21:01:41 +0200 Subject: [PATCH 5/7] Python script for creating json --- ci/create_nightly_json.py | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 ci/create_nightly_json.py diff --git a/ci/create_nightly_json.py b/ci/create_nightly_json.py new file mode 100644 index 000000000..4baffe365 --- /dev/null +++ b/ci/create_nightly_json.py @@ -0,0 +1,46 @@ +import subprocess +import sys +import json +import datetime; + +def main(): + files_by_date = {} + + files_lines = execute_cli("b2 ls --long odin-binaries nightly").split("\n") + for x in files_lines: + parts = x.split(" ", 1) + if parts[0]: + json_str = execute_cli(f"b2 get-file-info {parts[0]}") + data = json.loads(json_str) + url = execute_cli(f"b2 make-url {data['fileId']}").strip() + sha1 = data['contentSha1'] + name = remove_prefix(data['fileName'], "nightly/") + ts = int(data['fileInfo']['src_last_modified_millis']) + date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d') + + if date not in files_by_date.keys(): + files_by_date[date] = [] + + files_by_date[date].append({ + 'name': name, + 'url': url, + 'sha1': sha1, + }) + + now = datetime.datetime.utcnow().isoformat() + + print(json.dumps({ + 'last_updated' : now, + 'files': files_by_date + }, sort_keys=True, indent=4)) + +def remove_prefix(text, prefix): + return text[text.startswith(prefix) and len(prefix):] + +def execute_cli(command): + sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + return sb.stdout.read().decode("utf-8"); + +if __name__ == '__main__': + sys.exit(main()) + From 9655b61c11128fb29bea5c5a35b39393c70a718e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20Hjortsh=C3=B8j?= Date: Sat, 11 Apr 2020 21:08:50 +0200 Subject: [PATCH 6/7] Setup nightly.yml to run everyday at 20:00 UTC --- .github/workflows/nightly.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7d67d370d..1e1be0963 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,5 +1,7 @@ name: Nightly -on: [push] +on: + schedule: + - cron: 0 20 * * * jobs: build_windows: @@ -18,7 +20,7 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat - ./build_ci.bat + ./ci/build_ci.bat - name: Odin run shell: cmd run: | @@ -88,6 +90,7 @@ jobs: runs-on: [ubuntu-latest] needs: [build_windows, build_macos, build_ubuntu] steps: + - uses: actions/checkout@v1 - name: Install B2 CLI shell: bash run: sudo pip install --upgrade b2 @@ -103,14 +106,12 @@ jobs: uses: actions/download-artifact@v1 with: name: macos_artifacts - - name: List artifacts - run: find . -type f -printf "%T@ %p\n" | sort -nr | cut -d\ -f2- - name: Create archieves run: | now=$(date +'%Y-%m-%d') - 7z a test/odin-windows-amd64-nightly+$now.zip windows_artifacts/* - 7z a test/odin-ubuntu-amd64-nightly+$now.zip ubuntu_artifacts/* - 7z a test/odin-macos-amd64-nightly+$now.zip macos_artifacts/* + 7z a output/odin-windows-amd64-nightly+$now.zip -r windows_artifacts/ + 7z a output/odin-ubuntu-amd64-nightly+$now.zip -r ubuntu_artifacts/ + 7z a output/odin-macos-amd64-nightly+$now.zip -r macos_artifact/ - name: Upload artifacts to b2 shell: bash env: @@ -118,6 +119,8 @@ jobs: APPKEY: ${{ secrets.B2_APPKEY }} run: | b2 authorize-account "$APPID" "$APPKEY" - b2 sync --delete test b2://odin-binaries/nightly + b2 sync --keepDays 7 output b2://odin-binaries/nightly + python3 ci/create_nightly_json.py > nightly.json + b2 upload-file odin-binaries nightly.json nightly.json b2 clear-account From 8f2b84869876c33ce6d86e5a6b3acbf4c3465668 Mon Sep 17 00:00:00 2001 From: Mikkel Hjortshoej Date: Sat, 11 Apr 2020 21:48:44 +0200 Subject: [PATCH 7/7] Make the json for nightly have friendly URLs --- ci/create_nightly_json.py | 7 ++++--- ci/create_nightly_json.sh | 5 ----- 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 ci/create_nightly_json.sh diff --git a/ci/create_nightly_json.py b/ci/create_nightly_json.py index 4baffe365..43193a65d 100644 --- a/ci/create_nightly_json.py +++ b/ci/create_nightly_json.py @@ -1,7 +1,8 @@ import subprocess import sys import json -import datetime; +import datetime +import urllib.parse def main(): files_by_date = {} @@ -12,9 +13,9 @@ def main(): if parts[0]: json_str = execute_cli(f"b2 get-file-info {parts[0]}") data = json.loads(json_str) - url = execute_cli(f"b2 make-url {data['fileId']}").strip() - sha1 = data['contentSha1'] name = remove_prefix(data['fileName'], "nightly/") + url = f"https://f001.backblazeb2.com/file/odin-binaries/nightly/{urllib.parse.quote_plus(name)}" + sha1 = data['contentSha1'] ts = int(data['fileInfo']['src_last_modified_millis']) date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d') diff --git a/ci/create_nightly_json.sh b/ci/create_nightly_json.sh deleted file mode 100644 index 184423195..000000000 --- a/ci/create_nightly_json.sh +++ /dev/null @@ -1,5 +0,0 @@ -FILE_IDS=$(b2 ls --long odin-binaries nightly | cut -d ' ' -f 1) - -while IFS= read -r line; do - echo "... $line ..." -done <<< "$list" \ No newline at end of file