From 2575ce049eb4a3dbaac6a72fa9103379b8877a62 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 24 Nov 2023 20:15:27 +0100 Subject: [PATCH 01/27] Fetch generator --- get-generator-from-pr.py | 107 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 get-generator-from-pr.py diff --git a/get-generator-from-pr.py b/get-generator-from-pr.py new file mode 100644 index 0000000..fe69e3b --- /dev/null +++ b/get-generator-from-pr.py @@ -0,0 +1,107 @@ +import requests +import pprint +import re +import sys +import zipfile +import os +import platform + +""" Download djinni generator from the last successful check of a pull request. + + Usage: python get-generator-from-pr.py + Example: python get-generator-from-pr.py 123 + + It's unfortunately not possible to download artifacts from a pull request directly. + So there are a few steps involved: + 1. Get the commits for the pull request + 2. Get the checks for the last commit + 3. Find the last successful check + 4. Extract the workflow run ID from the details URL + 5. Get the list of artifacts for the workflow run + 6. Get the ID of the last artifact + 7. Download the last artifact + 8. Unzip the file + 9. Make the file executable (on Linux and Mac) + 10. Run the generator +""" + + +# For 'debugging' the JSON response +def dbg_print(data): + print('------------------') + pprint.pprint(data) + print('------------------') + + +# Get the token from the environment variable +TOKEN_ENV_VAR='GENERATOR_DL_TOKEN' +TOKEN = os.getenv(TOKEN_ENV_VAR) +if TOKEN is None: + sys.exit(f'{TOKEN_ENV_VAR} environment variable not set') + +# Metadata for the GitHub API +headers = {'Authorization': f'token {TOKEN}'} +owner = 'cross-language-cpp' +repo = 'djinni-generator' +# Get the PR number from command line arguments +pr_number = sys.argv[1] +download_name=f'djinni-generator-{pr_number}.zip' + +# Get the commits for the pull request +response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/commits', headers=headers) +commits = response.json() + +# Get the SHA of the last commit +last_commit_sha = commits[-1]['sha'] + +# Get the checks for the last commit +response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/commits/{last_commit_sha}/check-runs', headers=headers) +checks = response.json() + +# Find the last successful check +last_successful_check = None +for check in checks['check_runs']: + if check['status'] == 'completed' and check['conclusion'] == 'success': + last_successful_check = check + break + +if last_successful_check: + # Extract the workflow run ID from the details URL + match = re.search(r'/actions/runs/(\d+)', last_successful_check['details_url']) + if match: + run_id = match.group(1) + + # Get the list of artifacts for the workflow run + response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', headers=headers) + artifacts = response.json() + + if 'artifacts' in artifacts and artifacts['artifacts']: + # Get the ID of the last artifact + last_artifact_id = artifacts['artifacts'][0]['id'] + + # Download the last artifact + response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/actions/artifacts/{last_artifact_id}/zip', headers=headers) + + # Save the artifact to a file + with open(download_name, 'wb') as f: + f.write(response.content) + # Unzip the file + with zipfile.ZipFile(download_name, 'r') as zip_ref: + zip_ref.extractall('.') + # Get the name of the file in the zip + file_name = zip_ref.namelist()[0] + # Check if the script is running on Windows + if platform.system() == 'Windows': + # Add a .bat extension to the file + new_file_name = file_name + '.bat' + os.rename(file_name, new_file_name) + file_name = new_file_name + else: + # Make the file executable + os.chmod(file_name, 0o755) + else: + print('No artifacts found for this workflow run') + else: + print('Could not extract workflow run ID from details URL') +else: + print('No successful checks found') From a9d4fbbd8ef5fe251b1651706cae772fa15981e2 Mon Sep 17 00:00:00 2001 From: a4z Date: Thu, 21 Dec 2023 22:40:46 +0100 Subject: [PATCH 02/27] Add get-generator script --- get-generator-from-pr.py | 107 -------------------------- get-generator.py | 162 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 107 deletions(-) delete mode 100644 get-generator-from-pr.py create mode 100644 get-generator.py diff --git a/get-generator-from-pr.py b/get-generator-from-pr.py deleted file mode 100644 index fe69e3b..0000000 --- a/get-generator-from-pr.py +++ /dev/null @@ -1,107 +0,0 @@ -import requests -import pprint -import re -import sys -import zipfile -import os -import platform - -""" Download djinni generator from the last successful check of a pull request. - - Usage: python get-generator-from-pr.py - Example: python get-generator-from-pr.py 123 - - It's unfortunately not possible to download artifacts from a pull request directly. - So there are a few steps involved: - 1. Get the commits for the pull request - 2. Get the checks for the last commit - 3. Find the last successful check - 4. Extract the workflow run ID from the details URL - 5. Get the list of artifacts for the workflow run - 6. Get the ID of the last artifact - 7. Download the last artifact - 8. Unzip the file - 9. Make the file executable (on Linux and Mac) - 10. Run the generator -""" - - -# For 'debugging' the JSON response -def dbg_print(data): - print('------------------') - pprint.pprint(data) - print('------------------') - - -# Get the token from the environment variable -TOKEN_ENV_VAR='GENERATOR_DL_TOKEN' -TOKEN = os.getenv(TOKEN_ENV_VAR) -if TOKEN is None: - sys.exit(f'{TOKEN_ENV_VAR} environment variable not set') - -# Metadata for the GitHub API -headers = {'Authorization': f'token {TOKEN}'} -owner = 'cross-language-cpp' -repo = 'djinni-generator' -# Get the PR number from command line arguments -pr_number = sys.argv[1] -download_name=f'djinni-generator-{pr_number}.zip' - -# Get the commits for the pull request -response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/commits', headers=headers) -commits = response.json() - -# Get the SHA of the last commit -last_commit_sha = commits[-1]['sha'] - -# Get the checks for the last commit -response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/commits/{last_commit_sha}/check-runs', headers=headers) -checks = response.json() - -# Find the last successful check -last_successful_check = None -for check in checks['check_runs']: - if check['status'] == 'completed' and check['conclusion'] == 'success': - last_successful_check = check - break - -if last_successful_check: - # Extract the workflow run ID from the details URL - match = re.search(r'/actions/runs/(\d+)', last_successful_check['details_url']) - if match: - run_id = match.group(1) - - # Get the list of artifacts for the workflow run - response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', headers=headers) - artifacts = response.json() - - if 'artifacts' in artifacts and artifacts['artifacts']: - # Get the ID of the last artifact - last_artifact_id = artifacts['artifacts'][0]['id'] - - # Download the last artifact - response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/actions/artifacts/{last_artifact_id}/zip', headers=headers) - - # Save the artifact to a file - with open(download_name, 'wb') as f: - f.write(response.content) - # Unzip the file - with zipfile.ZipFile(download_name, 'r') as zip_ref: - zip_ref.extractall('.') - # Get the name of the file in the zip - file_name = zip_ref.namelist()[0] - # Check if the script is running on Windows - if platform.system() == 'Windows': - # Add a .bat extension to the file - new_file_name = file_name + '.bat' - os.rename(file_name, new_file_name) - file_name = new_file_name - else: - # Make the file executable - os.chmod(file_name, 0o755) - else: - print('No artifacts found for this workflow run') - else: - print('Could not extract workflow run ID from details URL') -else: - print('No successful checks found') diff --git a/get-generator.py b/get-generator.py new file mode 100644 index 0000000..b31dc28 --- /dev/null +++ b/get-generator.py @@ -0,0 +1,162 @@ +import requests +import pprint +import re +import sys +import zipfile +import os +import platform + +""" Download djinni generator from the last successful check of a pull request. + + Usage: python get-generator-from-pr.py + Example: python get-generator-from-pr.py 123 + + It's unfortunately not possible to download artifacts from a pull request directly. + So there are a few steps involved: + 1. Get the commits for the pull request + 2. Get the checks for the last commit + 3. Find the last successful check + 4. Extract the workflow run ID from the details URL + 5. Get the list of artifacts for the workflow run + 6. Get the ID of the last artifact + 7. Download the last artifact + 8. Unzip the file + 9. Make the file executable (on Linux and Mac) + 10. Run the generator +""" + +# Get the token from the environment variable +TOKEN_ENV_VAR='GENERATOR_DL_TOKEN' +TOKEN = os.getenv(TOKEN_ENV_VAR) +if TOKEN is None: + sys.exit(f'{TOKEN_ENV_VAR} environment variable not set') + +output_dir = 'zz' + + +# For 'debugging' the JSON response +def dbg_print(data): + print('------------------') + pprint.pprint(data) + print('------------------') + + + +def download_file(url, outdir, filename, headers=None): + + if not os.path.exists(outdir): + os.mkdir(outdir) + # print(f'Downloading from URL: {url}') + response = requests.get(url, stream=True, headers=headers) + # print(f'Response status code: {response.status_code}') + response.raise_for_status() + full_path = os.path.join(outdir, filename) + with open(full_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + # print(f'File downloaded successfully: {filename}') + return full_path + + +def make_executable(path): + if platform.system() == 'Windows': + # Add a .bat extension to the file + new_file_name = path + '.bat' + os.rename(path, new_file_name) + path = new_file_name + else: + # Make the file executable + os.chmod(path, 0o755) + return path + + +def download_from_pr(owner, repo, pr_number): + + # Metadata for the GitHub API + headers = {'Authorization': f'token {TOKEN}'} + download_name=f'djinni-generator-{pr_number}.zip' + + # Get the commits for the pull request + response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/commits', headers=headers) + commits = response.json() + + # Get the SHA of the last commit + last_commit_sha = commits[-1]['sha'] + + # Get the checks for the last commit + response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/commits/{last_commit_sha}/check-runs', headers=headers) + checks = response.json() + + # Find the last successful check + last_successful_check = None + for check in checks['check_runs']: + if check['status'] == 'completed' and check['conclusion'] == 'success': + last_successful_check = check + break + + if last_successful_check: + # Extract the workflow run ID from the details URL + match = re.search(r'/actions/runs/(\d+)', last_successful_check['details_url']) + if match: + run_id = match.group(1) + # Get the list of artifacts for the workflow run + response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', headers=headers) + artifacts = response.json() + + if 'artifacts' in artifacts and artifacts['artifacts']: + # Get the ID of the last artifact + last_artifact_id = artifacts['artifacts'][0]['id'] + url = f'https://api.github.com/repos/{owner}/{repo}/actions/artifacts/{last_artifact_id}/zip' + dl_file = download_file(url, output_dir, download_name, headers=headers) + + with zipfile.ZipFile(dl_file, 'r') as zip_ref: + zip_ref.extractall(output_dir) + file_name = zip_ref.namelist()[0] + + make_executable(os.path.join(output_dir,file_name)) + + else: + raise Exception('No artifacts found for this workflow run') + else: + raise Exception('Could not extract workflow run ID from details URL') + else: + raise Exception('No successful checks found') + +def get_latest_release(owner, repo): + response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/releases/latest') + response.raise_for_status() + download_url = response.json()['assets'][0]['browser_download_url'] + dl_file = download_file(download_url, output_dir, 'djinni') + make_executable(dl_file) + +def get_latest_pre_release(owner, repo): + response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/releases') + response.raise_for_status() + for release in response.json(): + if release['prerelease']: + download_url = release['assets'][0]['browser_download_url'] + print(download_url) + dl_file = download_file(download_url, output_dir, 'djinni') + make_executable(dl_file) + break + +# Get the argument, TODO, use argparse and add optional a folder argument +dl_arg = sys.argv[1] +if not dl_arg: + sys.exit('No argument given') + +owner = 'cross-language-cpp' +repo = 'djinni-generator' + +try: + if dl_arg.startswith("pr-"): + pr_number = dl_arg[len("pr-"):] + download_from_pr(owner, repo, pr_number) + elif dl_arg == 'latest': + get_latest_pre_release(owner, repo) + elif dl_arg == 'release': + get_latest_release(owner, repo) + else: + raise Exception('Invalid argument given') +except Exception as e: + sys.exit(f'Error occurred: {e}') From 1416980fc40913955faf1cbdc479f82200a86a1a Mon Sep 17 00:00:00 2001 From: a4z Date: Thu, 21 Dec 2023 22:41:04 +0100 Subject: [PATCH 03/27] Add request for get-generator --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index fc947f6..704a03a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# python bindings cffi==1.14.5 future==0.18.2 pytest==6.2.5 +# for local helpers +requests + From 3a0701ab31dd7e434e1ecf9ec5853cfb4f2ea7e2 Mon Sep 17 00:00:00 2001 From: a4z Date: Thu, 21 Dec 2023 22:41:33 +0100 Subject: [PATCH 04/27] First try for getting a generator without asdf --- .github/workflows/build.yml | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 195b0db..08a1cdd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,3 +1,13 @@ +env: + GENERATOR_BUILID: "release" + # valid options: "release", "latest", "pr-NNN" + # "release" is the latest djinni generator release + # "latest" is the djinni generator build of the latest state of main, might be equal release, or newer + # "pr-NNN" is the djinni generator build of a pull request, NNN is the pullrequest number + # This can be used to check that a support lib builds with new generator changes that have not been released yet + # Note: the pipeline will not succeed with a generator different than release, + # but we can verify that the support lib builds and tests successfully with a generator different (newer) than release + name: build-all-configs on: [push, pull_request] jobs: @@ -28,10 +38,10 @@ jobs: runs-on: macos-11 strategy: matrix: - python-version: ["3.7.12", "3.10.4"] # oldest and newest, rest assumed to work + python-version: ["3.7.17", "3.12.1"] # oldest and newest, rest assumed to work steps: - uses: actions/checkout@v2 - - uses: asdf-vm/actions/install@v1 + # - uses: asdf-vm/actions/install@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -40,10 +50,12 @@ jobs: uses: py-actions/py-dependency-install@v3 with: path: "requirements.txt" + - name: Download Genearator + run: python get-generator.py ${{ env.GENERATOR_BUILID }} - name: Report cmake version run: cmake --version - name: Configure cmake - run: cmake -S . -B build -DDJINNI_WITH_PYTHON=ON + run: cmake -S . -B build -DDJINNI_WITH_PYTHON=ON -DDJINNI_EXECUTABLE="$(pwd)/zz/djinni" - name: Build release run: cmake --build build --parallel $(sysctl -n hw.ncpu) --config Release - name: Run tests @@ -154,3 +166,14 @@ jobs: working-directory: build/check_install_root run: if((Compare-Object (Get-Content ..\..\test\cppcli_list.txt) (Resolve-Path -Path (Get-ChildItem -Recurse).FullName -Relative))) { Write-Error "file list not equal" } + + check_generator_buildid: + needs: ["build-on-windows-for-cppcli", "build-on-ubuntu-for-python", "build-on-ubuntu-for-android_arm32-with-thread-attach", "build-on-ubuntu-for-jni-with-thread-attach", "build-on-ubuntu-for-jni", "build-on-osx-for-python", "build-on-osx-for-objectiveC"] + runs-on: ubuntu-latest + steps: + - name: Check GENERATOR_BUILID + run: | + if [[ "${{ env.GENERATOR_BUILID }}" != "release" ]]; then + echo "GENERATOR_BUILID is not set to 'release'" + exit 1 + fi From 6cdea4ed4d90b4551934563f1c0e283688ee9d26 Mon Sep 17 00:00:00 2001 From: a4z Date: Thu, 21 Dec 2023 22:52:13 +0100 Subject: [PATCH 05/27] Add token access --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08a1cdd..957f808 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,6 +51,8 @@ jobs: with: path: "requirements.txt" - name: Download Genearator + env: + GENERATOR_DL_TOKEN: ${{ secrets.GENERATOR_DL_TOKEN }} run: python get-generator.py ${{ env.GENERATOR_BUILID }} - name: Report cmake version run: cmake --version From f375c8733223ac3e3b785ac9ddd4903899895941 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 13:06:26 +0100 Subject: [PATCH 06/27] re-enable asdf for java --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 957f808..9344e77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: python-version: ["3.7.17", "3.12.1"] # oldest and newest, rest assumed to work steps: - uses: actions/checkout@v2 - # - uses: asdf-vm/actions/install@v1 + - uses: asdf-vm/actions/install@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: From 16e58f3c0d5d33085cbbcf399f19c9e7007827ea Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 13:50:46 +0100 Subject: [PATCH 07/27] Use djinni from pr 161 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9344e77..9fda7b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,5 @@ env: - GENERATOR_BUILID: "release" + GENERATOR_BUILID: "pr-161" # valid options: "release", "latest", "pr-NNN" # "release" is the latest djinni generator release # "latest" is the djinni generator build of the latest state of main, might be equal release, or newer @@ -41,7 +41,7 @@ jobs: python-version: ["3.7.17", "3.12.1"] # oldest and newest, rest assumed to work steps: - uses: actions/checkout@v2 - - uses: asdf-vm/actions/install@v1 +# - uses: asdf-vm/actions/install@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: From 2cc2ed93aa75fd6d99273f4bcbb40b7cb0b0beb6 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:05:46 +0100 Subject: [PATCH 08/27] Flip lines with env variable --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9fda7b8..7dc2606 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,9 +51,9 @@ jobs: with: path: "requirements.txt" - name: Download Genearator + run: python get-generator.py ${{ env.GENERATOR_BUILID }} env: GENERATOR_DL_TOKEN: ${{ secrets.GENERATOR_DL_TOKEN }} - run: python get-generator.py ${{ env.GENERATOR_BUILID }} - name: Report cmake version run: cmake --version - name: Configure cmake From 52b0c03b508a2a898829f19331dd5e1f21bebf93 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:14:52 +0100 Subject: [PATCH 09/27] Use python with 3 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7dc2606..f7765fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: with: path: "requirements.txt" - name: Download Genearator - run: python get-generator.py ${{ env.GENERATOR_BUILID }} + run: python3 get-generator.py ${{ env.GENERATOR_BUILID }} env: GENERATOR_DL_TOKEN: ${{ secrets.GENERATOR_DL_TOKEN }} - name: Report cmake version From 2163f84fbfad8827b68d295c31c238790df00f45 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:20:04 +0100 Subject: [PATCH 10/27] More debug output --- .github/workflows/build.yml | 8 ++++++-- get-generator.py | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7765fb..b2262ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,9 +50,13 @@ jobs: uses: py-actions/py-dependency-install@v3 with: path: "requirements.txt" - - name: Download Genearator - run: python3 get-generator.py ${{ env.GENERATOR_BUILID }} + - name: Download Generator + run: | + echo "GENERATOR_BUILID: $GENERATOR_BUILID" + echo "GENERATOR_DL_TOKEN: $GENERATOR_DL_TOKEN" + python get-generator.py $GENERATOR_BUILID env: + GENERATOR_BUILID: ${{ env.GENERATOR_BUILID }} GENERATOR_DL_TOKEN: ${{ secrets.GENERATOR_DL_TOKEN }} - name: Report cmake version run: cmake --version diff --git a/get-generator.py b/get-generator.py index b31dc28..6ab2aa9 100644 --- a/get-generator.py +++ b/get-generator.py @@ -150,11 +150,14 @@ def get_latest_pre_release(owner, repo): try: if dl_arg.startswith("pr-"): + print(f'Downloading from PR: {dl_arg}') pr_number = dl_arg[len("pr-"):] download_from_pr(owner, repo, pr_number) elif dl_arg == 'latest': + print('Downloading latest') get_latest_pre_release(owner, repo) elif dl_arg == 'release': + print('Downloading release') get_latest_release(owner, repo) else: raise Exception('Invalid argument given') From e384335ad3e0d32f665c4787d72a09fb43ced9ce Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:21:38 +0100 Subject: [PATCH 11/27] More debug output, include error --- get-generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/get-generator.py b/get-generator.py index 6ab2aa9..cd78098 100644 --- a/get-generator.py +++ b/get-generator.py @@ -162,4 +162,5 @@ def get_latest_pre_release(owner, repo): else: raise Exception('Invalid argument given') except Exception as e: + print('Error:', e) sys.exit(f'Error occurred: {e}') From fe57a66c6fcedb46db6ecf152b95b94c162f42e7 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:25:36 +0100 Subject: [PATCH 12/27] More debug output --- .github/workflows/build.yml | 2 -- get-generator.py | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2262ca..274ad6a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,8 +52,6 @@ jobs: path: "requirements.txt" - name: Download Generator run: | - echo "GENERATOR_BUILID: $GENERATOR_BUILID" - echo "GENERATOR_DL_TOKEN: $GENERATOR_DL_TOKEN" python get-generator.py $GENERATOR_BUILID env: GENERATOR_BUILID: ${{ env.GENERATOR_BUILID }} diff --git a/get-generator.py b/get-generator.py index cd78098..7f13aba 100644 --- a/get-generator.py +++ b/get-generator.py @@ -6,6 +6,8 @@ import os import platform +import traceback + """ Download djinni generator from the last successful check of a pull request. Usage: python get-generator-from-pr.py @@ -162,5 +164,6 @@ def get_latest_pre_release(owner, repo): else: raise Exception('Invalid argument given') except Exception as e: - print('Error:', e) + print('Got Error:', e) + traceback.print_exc() sys.exit(f'Error occurred: {e}') From 450d519a36f91489c7c61b9419a9857727ad94fd Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:29:06 +0100 Subject: [PATCH 13/27] More debug output --- get-generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/get-generator.py b/get-generator.py index 7f13aba..0991979 100644 --- a/get-generator.py +++ b/get-generator.py @@ -81,6 +81,7 @@ def download_from_pr(owner, repo, pr_number): # Get the commits for the pull request response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/commits', headers=headers) commits = response.json() + dbg_print(commits) # Get the SHA of the last commit last_commit_sha = commits[-1]['sha'] From eefebb166eb960860f7a9c89711765efcef3d361 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:37:24 +0100 Subject: [PATCH 14/27] More debug output --- get-generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/get-generator.py b/get-generator.py index 0991979..1169349 100644 --- a/get-generator.py +++ b/get-generator.py @@ -78,6 +78,8 @@ def download_from_pr(owner, repo, pr_number): headers = {'Authorization': f'token {TOKEN}'} download_name=f'djinni-generator-{pr_number}.zip' + dbg_print(headers) + # Get the commits for the pull request response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/commits', headers=headers) commits = response.json() From 5556b5325e0883c0d935b0bcfa3a2c6df6db1383 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:41:02 +0100 Subject: [PATCH 15/27] More debug output --- get-generator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/get-generator.py b/get-generator.py index 1169349..5d7c8ae 100644 --- a/get-generator.py +++ b/get-generator.py @@ -33,6 +33,9 @@ if TOKEN is None: sys.exit(f'{TOKEN_ENV_VAR} environment variable not set') +print(f'Using token: {TOKEN}') +print(f'Using token with : {len(TOKEN)} chars') + output_dir = 'zz' From b2d6f8ef9caadf468469e9abbdc1c4dbae77e239 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 14:57:38 +0100 Subject: [PATCH 16/27] More debug output --- .github/workflows/build.yml | 2 +- get-generator.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 274ad6a..69a6072 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: path: "requirements.txt" - name: Download Generator run: | - python get-generator.py $GENERATOR_BUILID + python get-generator.py $GENERATOR_BUILID ${{ secrets.GENERATOR_DL_TOKEN }} env: GENERATOR_BUILID: ${{ env.GENERATOR_BUILID }} GENERATOR_DL_TOKEN: ${{ secrets.GENERATOR_DL_TOKEN }} diff --git a/get-generator.py b/get-generator.py index 5d7c8ae..72f0b68 100644 --- a/get-generator.py +++ b/get-generator.py @@ -27,6 +27,17 @@ 10. Run the generator """ +dl_arg = sys.argv[1] +if not dl_arg: + sys.exit('No argument given') + + +key_arg = sys.argv[2] + +if key_arg: + print(f'Using key: {key_arg}') + os.environ['GENERATOR_DL_TOKEN'] = key_arg + # Get the token from the environment variable TOKEN_ENV_VAR='GENERATOR_DL_TOKEN' TOKEN = os.getenv(TOKEN_ENV_VAR) @@ -148,10 +159,6 @@ def get_latest_pre_release(owner, repo): make_executable(dl_file) break -# Get the argument, TODO, use argparse and add optional a folder argument -dl_arg = sys.argv[1] -if not dl_arg: - sys.exit('No argument given') owner = 'cross-language-cpp' repo = 'djinni-generator' From b94efea876628d479f366af330f7f899d0195e3d Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 15:05:54 +0100 Subject: [PATCH 17/27] More debug output --- .github/workflows/build.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69a6072..9b44982 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,9 @@ env: # Note: the pipeline will not succeed with a generator different than release, # but we can verify that the support lib builds and tests successfully with a generator different (newer) than release + + + name: build-all-configs on: [push, pull_request] jobs: @@ -40,6 +43,11 @@ jobs: matrix: python-version: ["3.7.17", "3.12.1"] # oldest and newest, rest assumed to work steps: + - name: Test Secret Value + env: + TEST_SECRET: ${{ secrets.GENERATOR_DL_TOKEN }} + run: | + echo "Length of secret: ${#TEST_SECRET}" - uses: actions/checkout@v2 # - uses: asdf-vm/actions/install@v1 - name: Set up Python ${{ matrix.python-version }} @@ -69,6 +77,11 @@ jobs: build-on-ubuntu-for-jni: runs-on: ubuntu-latest steps: + - name: Test Secret Value + env: + TEST_SECRET: ${{ secrets.GENERATOR_DL_TOKEN }} + run: | + echo "Length of secret: ${#TEST_SECRET}" - uses: actions/checkout@v2 - uses: asdf-vm/actions/install@v1 - name: Report cmake version From 1e38df662fe1a99bc5160b2357d027260b0050c8 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 15:38:38 +0100 Subject: [PATCH 18/27] More debug output --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9b44982..7b42d41 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,8 +46,10 @@ jobs: - name: Test Secret Value env: TEST_SECRET: ${{ secrets.GENERATOR_DL_TOKEN }} + TEST_VAL: ${{ env.TEST_VAR }} run: | echo "Length of secret: ${#TEST_SECRET}" + echo "Length of val: ${#TEST_VAL}" - uses: actions/checkout@v2 # - uses: asdf-vm/actions/install@v1 - name: Set up Python ${{ matrix.python-version }} From e9ea265b7017e8fc55fa6563a99bdce4823ba79c Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 15:41:00 +0100 Subject: [PATCH 19/27] More debug output --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b42d41..56e87ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: - name: Test Secret Value env: TEST_SECRET: ${{ secrets.GENERATOR_DL_TOKEN }} - TEST_VAL: ${{ env.TEST_VAR }} + TEST_VAL: ${{ vars.TEST_VAR }} run: | echo "Length of secret: ${#TEST_SECRET}" echo "Length of val: ${#TEST_VAL}" From dafc7a274668cc7456c2bcd0b5599f854ef0092f Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 15:49:58 +0100 Subject: [PATCH 20/27] More debug output --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 56e87ee..24b5c69 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ env: name: build-all-configs -on: [push, pull_request] +on: [push, pull_request_target, workflow_dispatch] jobs: build-on-osx-for-objectiveC: runs-on: macos-11 From 9533e6c49ea811ff1068dce6516cd15491fa7c4a Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 15:53:27 +0100 Subject: [PATCH 21/27] More debug output --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 24b5c69..1fb85fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ env: name: build-all-configs -on: [push, pull_request_target, workflow_dispatch] +on: [push, pull_request, workflow_dispatch] jobs: build-on-osx-for-objectiveC: runs-on: macos-11 From f5506308294136e54c75780990f789a5a889ecfe Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 15:55:19 +0100 Subject: [PATCH 22/27] More debug output --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1fb85fb..24b5c69 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ env: name: build-all-configs -on: [push, pull_request, workflow_dispatch] +on: [push, pull_request_target, workflow_dispatch] jobs: build-on-osx-for-objectiveC: runs-on: macos-11 From 6bb0c35be7093a1b7e7da8b6fb69b6e36cbb8a41 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 16:01:57 +0100 Subject: [PATCH 23/27] More debug output --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 24b5c69..cc9dc5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,11 +8,8 @@ env: # Note: the pipeline will not succeed with a generator different than release, # but we can verify that the support lib builds and tests successfully with a generator different (newer) than release - - - name: build-all-configs -on: [push, pull_request_target, workflow_dispatch] +on: [push, pull_request] jobs: build-on-osx-for-objectiveC: runs-on: macos-11 From 574fe9e3406d30b4cc50e776b0fcef7cee7d2025 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 16:16:49 +0100 Subject: [PATCH 24/27] More debug output --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc9dc5c..edc284f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,7 +62,7 @@ jobs: python get-generator.py $GENERATOR_BUILID ${{ secrets.GENERATOR_DL_TOKEN }} env: GENERATOR_BUILID: ${{ env.GENERATOR_BUILID }} - GENERATOR_DL_TOKEN: ${{ secrets.GENERATOR_DL_TOKEN }} + GENERATOR_DL_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Report cmake version run: cmake --version - name: Configure cmake From ffabbab2ec18b6987dfe2c865bfe459d8c61c4e0 Mon Sep 17 00:00:00 2001 From: a4z Date: Fri, 22 Dec 2023 16:19:23 +0100 Subject: [PATCH 25/27] More debug output --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index edc284f..8b4c8dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Test Secret Value env: - TEST_SECRET: ${{ secrets.GENERATOR_DL_TOKEN }} + TEST_SECRET: ${{ secrets.GITHUB_TOKEN }} TEST_VAL: ${{ vars.TEST_VAR }} run: | echo "Length of secret: ${#TEST_SECRET}" @@ -59,7 +59,7 @@ jobs: path: "requirements.txt" - name: Download Generator run: | - python get-generator.py $GENERATOR_BUILID ${{ secrets.GENERATOR_DL_TOKEN }} + python get-generator.py $GENERATOR_BUILID env: GENERATOR_BUILID: ${{ env.GENERATOR_BUILID }} GENERATOR_DL_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 5fff0f660b0134a8b25b2058eb99787e3cf123dd Mon Sep 17 00:00:00 2001 From: a4z Date: Sat, 23 Dec 2023 19:55:07 +0100 Subject: [PATCH 26/27] Re-try pr target --- .github/workflows/build.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8b4c8dc..3f0beda 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,15 @@ env: # but we can verify that the support lib builds and tests successfully with a generator different (newer) than release name: build-all-configs -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + branches: + - main + pull_request_target: + types: [assigned, opened, synchronize, reopened] jobs: build-on-osx-for-objectiveC: runs-on: macos-11 From 171e31b6433e9035d8db94c921c3c402ccc23a16 Mon Sep 17 00:00:00 2001 From: a4z Date: Sat, 23 Dec 2023 19:58:32 +0100 Subject: [PATCH 27/27] Re-try pr target 1 --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f0beda..79049d7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,9 +13,9 @@ on: push: branches: - main - pull_request: - branches: - - main + # pull_request: + # branches: + # - main pull_request_target: types: [assigned, opened, synchronize, reopened] jobs: