From 6814cdbee6cc99d9c6703fef4a2a7d8c46b29521 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 2 Jun 2020 05:47:07 +0000 Subject: [PATCH 01/26] Use base_python as package name --- build_python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_python.py b/build_python.py index 1a9772c..86e2493 100755 --- a/build_python.py +++ b/build_python.py @@ -5,7 +5,7 @@ import os from pathlib import Path -package_name = 'python' +package_name = 'base_python' python_version = '3.7.7' macos_deployment_target = '10.15' @@ -51,9 +51,9 @@ def output_base_name(): def python_destdir(): if windows(): - return Path('D:\\x_mirror\\buildman\\tools\\Python') + return Path('D:\\x_mirror\\buildman\\tools\\base_python') else: - return Path('/opt/ccdc/third-party/python') + return Path('/opt/ccdc/third-party/base_python') def python_version_destdir(): return python_destdir() / output_base_name() From 329b72c120d54b847d6ff8cd736b5f3b151c8368 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 2 Jun 2020 05:52:34 +0000 Subject: [PATCH 02/26] Updated common-tasks.yml --- common-tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-tasks.yml b/common-tasks.yml index 29d0f99..6a4b23e 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -33,7 +33,7 @@ steps: { "files": [ { - "pattern": "$(Build.ArtifactStagingDirectory)/python*", + "pattern": "$(Build.ArtifactStagingDirectory)/base_python*", "target": "ccdc-3rdparty-python-interpreters" } ] From cf7f6a4c4545684c75ae559ad19b73d496bed551 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 2 Jun 2020 06:08:02 +0000 Subject: [PATCH 03/26] Updated build_python.py --- build_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_python.py b/build_python.py index 86e2493..a700887 100755 --- a/build_python.py +++ b/build_python.py @@ -6,7 +6,7 @@ from pathlib import Path package_name = 'base_python' -python_version = '3.7.7' +python_version = sys.args[0] macos_deployment_target = '10.15' def macos(): From a98f493f4034c11a4e1ef31e96e41563efa8faf0 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 2 Jun 2020 06:09:55 +0000 Subject: [PATCH 04/26] many pythons --- common-tasks.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/common-tasks.yml b/common-tasks.yml index 6a4b23e..ee20ec3 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -4,7 +4,22 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - displayName: 'install python' + arguments: 3.6.10 + displayName: 'install python 3.6.10' + +- task: PythonScript@0 + inputs: + scriptSource: 'filePath' + scriptPath: build_python.py + arguments: 3.7.7 + displayName: 'install python 3.7.7' + +- task: PythonScript@0 + inputs: + scriptSource: 'filePath' + scriptPath: build_python.py + arguments: 3.8.3 + displayName: 'install python 3.8.3' # Upload artifactory build info - task: ArtifactoryCollectIssues@1 From 62ebaf6f348d133f0e2c4fbe103efc086f05b0f9 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 2 Jun 2020 06:11:13 +0000 Subject: [PATCH 05/26] Updated build_python.py --- build_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_python.py b/build_python.py index a700887..6811e37 100755 --- a/build_python.py +++ b/build_python.py @@ -6,7 +6,7 @@ from pathlib import Path package_name = 'base_python' -python_version = sys.args[0] +python_version = sys.argv[1] macos_deployment_target = '10.15' def macos(): From 5651d62ce8eee12e984c0105c4f31e4711f08ba5 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 2 Jun 2020 06:13:29 +0000 Subject: [PATCH 06/26] Updated common-tasks.yml --- common-tasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common-tasks.yml b/common-tasks.yml index ee20ec3..db46aa8 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -4,8 +4,8 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.6.10 - displayName: 'install python 3.6.10' + arguments: 3.6.8 #later versions don't have installers for some reason + displayName: 'install python 3.6.8' - task: PythonScript@0 inputs: From e2287d6d090d342a320be2f11ce9ee1939bd6cd0 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 2 Jun 2020 06:35:17 +0000 Subject: [PATCH 07/26] Updated common-tasks.yml --- common-tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-tasks.yml b/common-tasks.yml index db46aa8..5f9e494 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -18,7 +18,7 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.8.3 + arguments: 3.8.2 # pyenv doesn't carry 3.8.3 yet displayName: 'install python 3.8.3' # Upload artifactory build info From 95960654301b1078e4c50cd37b4e1a1dba974637 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Sat, 31 Oct 2020 18:52:32 +0000 Subject: [PATCH 08/26] Enable shared python library in linux --- build_python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_python.py b/build_python.py index 6811e37..866ac50 100755 --- a/build_python.py +++ b/build_python.py @@ -105,6 +105,7 @@ def install_pyenv_version(version): python_build_env['PYTHON_CONFIGURE_OPTS']="--with-tcltk-includes='-I/usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'" if linux(): python_build_env['PATH']=f"/tmp/pyenvinst/plugins/python-build/bin:{python_build_env['PATH']}" + python_build_env['PYTHON_CONFIGURE_OPTS']="--enable-shared" subprocess.run(f'sudo env "PATH=$PATH" python-build {version} {python_version_destdir()}', shell=True, check=True, env=python_build_env) From 9ee5b3e2ca8cc80cce90408e358aef61d2c0de1a Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Sat, 31 Oct 2020 18:53:00 +0000 Subject: [PATCH 09/26] update to python 3.7.9, 3.8.6, 3.9.0 --- common-tasks.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/common-tasks.yml b/common-tasks.yml index 5f9e494..7cb4de8 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -11,15 +11,22 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.7.7 - displayName: 'install python 3.7.7' + arguments: 3.7.9 + displayName: 'install python 3.7.9' - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.8.2 # pyenv doesn't carry 3.8.3 yet - displayName: 'install python 3.8.3' + arguments: 3.8.6 # added in pyenv v1.2.21 + displayName: 'install python 3.8.6' + +- task: PythonScript@0 + inputs: + scriptSource: 'filePath' + scriptPath: build_python.py + arguments: 3.9.0 # added in pyenv v1.2.21 + displayName: 'install python 3.9.0' # Upload artifactory build info - task: ArtifactoryCollectIssues@1 From 1829741c7bd129e44f0a2ef054028d5877218b01 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 2 Nov 2020 08:55:36 +0000 Subject: [PATCH 10/26] Try reordering as windows builds fail due to path changes --- common-tasks.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common-tasks.yml b/common-tasks.yml index 7cb4de8..e7d9f68 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -18,15 +18,15 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.8.6 # added in pyenv v1.2.21 - displayName: 'install python 3.8.6' + arguments: 3.9.0 # added in pyenv v1.2.21 + displayName: 'install python 3.9.0' -- task: PythonScript@0 + - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.9.0 # added in pyenv v1.2.21 - displayName: 'install python 3.9.0' + arguments: 3.8.6 # added in pyenv v1.2.21 + displayName: 'install python 3.8.6' # Upload artifactory build info - task: ArtifactoryCollectIssues@1 From 5b7d44b5abf03df06449faaefb868429b0758725 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 2 Nov 2020 08:56:58 +0000 Subject: [PATCH 11/26] Fix whitespace --- common-tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-tasks.yml b/common-tasks.yml index e7d9f68..cb0cccd 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -21,7 +21,7 @@ steps: arguments: 3.9.0 # added in pyenv v1.2.21 displayName: 'install python 3.9.0' - - task: PythonScript@0 +- task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py From 7bb2824dd63bb05dbe80a7b9db33b9ed8ccd1281 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Fri, 15 Jan 2021 00:00:39 +0000 Subject: [PATCH 12/26] update pyenv, hopefully build with tcl-tk --- build_python.py | 2 ++ common-tasks.yml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build_python.py b/build_python.py index 866ac50..5f62b34 100755 --- a/build_python.py +++ b/build_python.py @@ -78,6 +78,7 @@ def install_from_msi(): def install_prerequisites(): if macos(): + subprocess.run(['brew', 'update'], check=True) subprocess.run(['brew', 'install', 'openssl', 'readline', 'sqlite3', 'xz', 'zlib', 'tcl-tk'], check=True) if linux(): if centos(): @@ -100,6 +101,7 @@ def install_pyenv_version(version): if macos(): python_build_env['PATH']=f"/usr/local/opt/tcl-tk/bin:{python_build_env['PATH']}" python_build_env['LDFLAGS']=f"-L/usr/local/opt/tcl-tk/lib -mmacosx-version-min={macos_deployment_target}" + python_build_env['CFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" python_build_env['CPPFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" python_build_env['PKG_CONFIG_PATH']="/usr/local/opt/tcl-tk/lib/pkgconfig" python_build_env['PYTHON_CONFIGURE_OPTS']="--with-tcltk-includes='-I/usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'" diff --git a/common-tasks.yml b/common-tasks.yml index cb0cccd..de5ae82 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -18,15 +18,15 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.9.0 # added in pyenv v1.2.21 - displayName: 'install python 3.9.0' + arguments: 3.8.7 # added in pyenv v1.2.22 + displayName: 'install python 3.8.6' - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.8.6 # added in pyenv v1.2.21 - displayName: 'install python 3.8.6' + arguments: 3.9.1 # added in pyenv v1.2.22 + displayName: 'install python 3.9.0' # Upload artifactory build info - task: ArtifactoryCollectIssues@1 From 353965bf794dc454881971e9134c48ecb6e2db29 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Fri, 15 Jan 2021 08:37:47 +0000 Subject: [PATCH 13/26] use the "wrong" order again, on purpose --- common-tasks.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common-tasks.yml b/common-tasks.yml index de5ae82..fec8d2c 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -18,15 +18,17 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.8.7 # added in pyenv v1.2.22 - displayName: 'install python 3.8.6' + arguments: 3.9.1 # added in pyenv v1.2.22 + displayName: 'install python 3.9.1' +# Building 3.8 last on purpose because the installer on windows clobbers the system's hosted python +# Just keep it last or wonder why the system is loading python2 afterwards... - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.9.1 # added in pyenv v1.2.22 - displayName: 'install python 3.9.0' + arguments: 3.8.7 # added in pyenv v1.2.22 + displayName: 'install python 3.8.6' # Upload artifactory build info - task: ArtifactoryCollectIssues@1 From 3703710e110158b102097752f39c7ec6dc9dc2fd Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 25 Jan 2021 21:53:22 +0000 Subject: [PATCH 14/26] Run a basic smoke test to verify sqlite3 and tkinter modules load --- build_python.py | 10 ++++++++++ smoke_test.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 smoke_test.py diff --git a/build_python.py b/build_python.py index 5f62b34..155296b 100755 --- a/build_python.py +++ b/build_python.py @@ -58,6 +58,12 @@ def python_destdir(): def python_version_destdir(): return python_destdir() / output_base_name() +def python_interpreter(): + if windows(): + return python_version_destdir() / 'python.exe' + else + return python_version_destdir() / 'bin' / 'python' + def prepare_output_dir(): if linux(): subprocess.run(f'sudo mkdir -p {python_destdir()}', shell=True) @@ -114,6 +120,9 @@ def install_pyenv_version(version): def output_archive_filename(): return f'{output_base_name()}.tar.gz' +def smoke_test(): + subprocess.check_call([f'{ python_interpreter() }', 'smoke_test.py']) + def create_archive(): if 'BUILD_ARTIFACTSTAGINGDIRECTORY' in os.environ: archive_output_directory = Path( @@ -146,6 +155,7 @@ def main(): install_prerequisites() install_pyenv() install_pyenv_version(python_version) + smoke_test() create_archive() if __name__ == "__main__": diff --git a/smoke_test.py b/smoke_test.py new file mode 100644 index 0000000..2bd9bc4 --- /dev/null +++ b/smoke_test.py @@ -0,0 +1,15 @@ +import platform + +# there should be no issues importing sqlite libraries +import sqlite3 + +# pyenv has trouble building the tkinter extension, so checking for that +# this tends to happen on macos, where even the exception handling below +# will error out as _tkinter is not present +# On Linux, the import can fail because DISPLAY is not set. In this case +# a TclError is raised. But if we have that, we're good to go. +try: + import tkinter + print('ok') +except _tkinter.TclError: + print("no display, but that's ok") From 37f29ca0c7dbc1a1db11a6445f9791502617c8d4 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 25 Jan 2021 22:23:11 +0000 Subject: [PATCH 15/26] Fix syntax error --- build_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_python.py b/build_python.py index 155296b..efeea43 100755 --- a/build_python.py +++ b/build_python.py @@ -61,7 +61,7 @@ def python_version_destdir(): def python_interpreter(): if windows(): return python_version_destdir() / 'python.exe' - else + else: return python_version_destdir() / 'bin' / 'python' def prepare_output_dir(): From 63847bbf99471f3526232e9dac789a590826c62a Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 25 Jan 2021 23:47:12 +0000 Subject: [PATCH 16/26] Update sudo command to preserve environment (Fixes tk on macos) --- build_python.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build_python.py b/build_python.py index efeea43..82987cc 100755 --- a/build_python.py +++ b/build_python.py @@ -105,17 +105,21 @@ def install_pyenv(): def install_pyenv_version(version): python_build_env = dict(os.environ) if macos(): + python_build_env['MACOSX_DEPLOYMENT_TARGET']=f'{macos_deployment_target}' python_build_env['PATH']=f"/usr/local/opt/tcl-tk/bin:{python_build_env['PATH']}" python_build_env['LDFLAGS']=f"-L/usr/local/opt/tcl-tk/lib -mmacosx-version-min={macos_deployment_target}" python_build_env['CFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" python_build_env['CPPFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" python_build_env['PKG_CONFIG_PATH']="/usr/local/opt/tcl-tk/lib/pkgconfig" - python_build_env['PYTHON_CONFIGURE_OPTS']="--with-tcltk-includes='-I/usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'" + python_build_env['PYTHON_CONFIGURE_OPTS']="--with-tcltk-includes='-I/usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6' --enable-shared" + + subprocess.run(f'sudo "--preserve-env=MACOSX_DEPLOYMENT_TARGET,PATH,LDFLAGS,CFLAGS,CPPFLAGS,PKG_CONFIG_PATH,PYTHON_CONFIGURE_OPTS" python-build -v {version} {python_version_destdir()}', shell=True, check=True, env=python_build_env) + if linux(): python_build_env['PATH']=f"/tmp/pyenvinst/plugins/python-build/bin:{python_build_env['PATH']}" python_build_env['PYTHON_CONFIGURE_OPTS']="--enable-shared" - subprocess.run(f'sudo env "PATH=$PATH" python-build {version} {python_version_destdir()}', shell=True, check=True, env=python_build_env) + subprocess.run(f'sudo "--preserve-env=PATH,PYTHON_CONFIGURE_OPTS" python-build -v {version} {python_version_destdir()}', shell=True, check=True, env=python_build_env) def output_archive_filename(): return f'{output_base_name()}.tar.gz' From 61bf2592ab7c9b2221aeb822e35a43fb3dce6955 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 25 Jan 2021 23:47:27 +0000 Subject: [PATCH 17/26] Improve smoke test message --- smoke_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/smoke_test.py b/smoke_test.py index 2bd9bc4..e3ac912 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -10,6 +10,7 @@ # a TclError is raised. But if we have that, we're good to go. try: import tkinter - print('ok') except _tkinter.TclError: print("no display, but that's ok") + +print('python interpreter smoke test ok') From fcd3608e0384971c668cfbe36fa810deb54f5770 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Tue, 26 Jan 2021 01:01:56 +0000 Subject: [PATCH 18/26] Fix path to pyenv build --- build_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_python.py b/build_python.py index 82987cc..da3d0fa 100755 --- a/build_python.py +++ b/build_python.py @@ -119,7 +119,7 @@ def install_pyenv_version(version): python_build_env['PATH']=f"/tmp/pyenvinst/plugins/python-build/bin:{python_build_env['PATH']}" python_build_env['PYTHON_CONFIGURE_OPTS']="--enable-shared" - subprocess.run(f'sudo "--preserve-env=PATH,PYTHON_CONFIGURE_OPTS" python-build -v {version} {python_version_destdir()}', shell=True, check=True, env=python_build_env) + subprocess.run(f'sudo "--preserve-env=PATH,PYTHON_CONFIGURE_OPTS" /tmp/pyenvinst/plugins/python-build/bin/python-build -v {version} {python_version_destdir()}', shell=True, check=True, env=python_build_env) def output_archive_filename(): return f'{output_base_name()}.tar.gz' From 51048e8b748053c6454c1078852f787add13a317 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 26 Apr 2021 09:28:10 +0100 Subject: [PATCH 19/26] Update base python versions --- common-tasks.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common-tasks.yml b/common-tasks.yml index fec8d2c..922364b 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -11,14 +11,14 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.7.9 - displayName: 'install python 3.7.9' + arguments: 3.7.10 + displayName: 'install python 3.7.10' - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.9.1 # added in pyenv v1.2.22 + arguments: 3.9.4 # added in pyenv v1.2.26 displayName: 'install python 3.9.1' # Building 3.8 last on purpose because the installer on windows clobbers the system's hosted python @@ -27,8 +27,8 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.8.7 # added in pyenv v1.2.22 - displayName: 'install python 3.8.6' + arguments: 3.8.9 # added in pyenv v1.2.25 + displayName: 'install python 3.8.9' # Upload artifactory build info - task: ArtifactoryCollectIssues@1 From 22450ed3b3a19c878105966ee8e632459387e696 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 26 Apr 2021 09:34:48 +0100 Subject: [PATCH 20/26] use latest versions with windows installers --- common-tasks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common-tasks.yml b/common-tasks.yml index 922364b..12aa6e8 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -4,15 +4,15 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.6.8 #later versions don't have installers for some reason + arguments: 3.6.8 #later versions don't have windows installers displayName: 'install python 3.6.8' - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.7.10 - displayName: 'install python 3.7.10' + arguments: 3.7.9 #later versions don't have windows installers + displayName: 'install python 3.7.9' - task: PythonScript@0 inputs: From 9a68223c7e0fc5aa40faf59fe878bd82f9b89d27 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 26 Apr 2021 09:57:58 +0100 Subject: [PATCH 21/26] Install local version of sqlite3 on linux --- build_python.py | 58 ++++- ccdc/thirdparty/package.py | 483 +++++++++++++++++++++++++++++++++++++ smoke_test.py | 4 + 3 files changed, 544 insertions(+), 1 deletion(-) create mode 100644 ccdc/thirdparty/package.py diff --git a/build_python.py b/build_python.py index da3d0fa..00394e4 100755 --- a/build_python.py +++ b/build_python.py @@ -4,11 +4,65 @@ import sys import os from pathlib import Path +from ccdc.thirdparty.package import Package, AutoconfMixin, MakeInstallMixin, NoArchiveMixin, CMakeMixin package_name = 'base_python' python_version = sys.argv[1] macos_deployment_target = '10.15' +class InstallInBasePythonMixin(object): + @property + def install_directory(self): + return python_version_destdir() + +class SqlitePackage(InstallInBasePythonMixin, AutoconfMixin, NoArchiveMixin, Package): + '''SQLite''' + name = 'sqlite' + version = '3.35.5' + tarversion = '3350500' + + @property + def source_archives(self): + return { + f'sqlite-autoconf-{self.tarversion}.tar.gz': f'https://www.sqlite.org/2021/sqlite-autoconf-{self.tarversion}.tar.gz' + } + + @property + def main_source_directory_path(self): + return self.source_extracted / f'{self.name}-autoconf-{self.tarversion}' + + @property + def cflags(self): + return super().cflags + [ + '-DSQLITE_ENABLE_FTS3', + '-DSQLITE_ENABLE_FTS3_PARENTHESIS', + '-DSQLITE_ENABLE_FTS4', + '-DSQLITE_ENABLE_FTS5', + '-DSQLITE_ENABLE_EXPLAIN_COMMENTS', + '-DSQLITE_ENABLE_NULL_TRIM', + '-DSQLITE_MAX_COLUMN=10000', + '-DSQLITE_ENABLE_JSON1', + '-DSQLITE_ENABLE_RTREE', + '-DSQLITE_TCL=0', + '-fPIC', + ] + + @property + def ldflags(self): + return super().ldflags + [ + '-lm' + ] + + @property + def arguments_to_configuration_script(self): + return super().arguments_to_configuration_script + [ + '--enable-threadsafe', + '--enable-shared=no', + '--enable-static=yes', + '--disable-readline', + '--disable-dependency-tracking', + ] + def macos(): return sys.platform == 'darwin' @@ -89,7 +143,9 @@ def install_prerequisites(): if linux(): if centos(): subprocess.run('sudo yum update -y', shell=True, check=True) - subprocess.run('sudo yum install -y findutils gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel tk-devel xz xz-devel libffi-devel', shell=True, check=True) + subprocess.run('sudo yum install -y findutils gcc zlib-devel bzip2 bzip2-devel readline-devel openssl-devel tk-devel xz xz-devel libffi-devel', shell=True, check=True) + # See https://jira.ccdc.cam.ac.uk/browse/BLD-5684 + SqlitePackage().build() if ubuntu(): subprocess.run('sudo apt-get -y update', shell=True, check=True) subprocess.run('sudo apt-get -y dist-upgrade', shell=True, check=True) diff --git a/ccdc/thirdparty/package.py b/ccdc/thirdparty/package.py new file mode 100644 index 0000000..f517b3e --- /dev/null +++ b/ccdc/thirdparty/package.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +import os +import stat +import shutil +import tempfile +import multiprocessing +import getpass +from pathlib import Path +from distutils.version import StrictVersion + + +class Package(object): + '''Base for anything installable''' + name = None + version = None + _cached_sdkroot = None + + def __init__(self): + self.use_vs_version_in_base_name = True + self.use_distribution_in_base_name = False + + @property + def macos(self): + return sys.platform == 'darwin' + + @property + def windows(self): + return sys.platform == 'win32' + + @property + def linux(self): + return sys.platform.startswith('linux') + + @property + def centos(self): + return self.linux and Path('/etc/centos-release').exists() + + @property + def centos_major_version(self): + return subprocess.check_output('rpm -E %{rhel}', shell=True).decode('utf-8').strip() + + @property + def debian(self): + return self.linux and Path('/etc/debian_version').exists() + + @property + def ubuntu(self): + return self.debian and subprocess.check_output('lsb_release -i -s', shell=True).decode('utf-8').strip() == 'Ubuntu' + + @property + def ubuntu_version(self): + return subprocess.check_output('lsb_release -r -s', shell=True).decode('utf-8').strip() + + @property + def platform(self): + if not self.use_distribution_in_base_name: + return sys.platform + if not self.linux: + return sys.platform + if self.centos: + return f'centos{self.centos_major_version}' + if self.ubuntu: + return f'ubuntu{self.ubuntu_version}' + + @property + def macos_sdkroot(self): + if not self.macos: + return None + if not self._cached_sdkroot: + self._cached_sdkroot = subprocess.check_output( + ['xcrun', '--show-sdk-path'])[:-1].decode('utf8') + return self._cached_sdkroot + + @property + def macos_deployment_target(self): + '''The minimum macos version the pagkage will work on''' + return '10.12' + + def prepare_directories(self): + if not self.toolbase.exists() and not self.windows: + subprocess.check_output(['sudo', 'mkdir', '-p', '/opt/ccdc']) + subprocess.check_output( + ['sudo', 'chown', f'{getpass.getuser()}', '/opt/ccdc']) + self.toolbase.mkdir(parents=True, exist_ok=True) + self.source_downloads_base.mkdir(parents=True, exist_ok=True) + self.source_extracted_base.mkdir(parents=True, exist_ok=True) + self.source_builds_base.mkdir(parents=True, exist_ok=True) + self.build_logs.mkdir(parents=True, exist_ok=True) + + @property + def toolbase(self): + '''Return the base directory where tools are installed''' + if self.windows: + return Path('D:\\x_mirror\\buildman\\tools') + else: + return Path('/opt/ccdc/third-party') + + @property + def source_downloads_base(self): + '''Return the directory where sources are downloaded''' + if self.windows: + if 'SYSTEM_ARTIFACTSDIRECTORY' in os.environ: + return Path(os.environ['SYSTEM_ARTIFACTSDIRECTORY']) + return Path('D:\\tp\\downloads') + else: + return Path('/opt/ccdc/third-party-sources/downloads') + + @property + def source_extracted_base(self): + '''Return the base directory where sources are extracted''' + if self.windows: + return Path('D:\\tp\\extracted') + else: + return Path('/opt/ccdc/third-party-sources/extracted') + + @property + def source_builds_base(self): + '''Return the base directory where sources are built''' + if self.windows: + return Path('D:\\tp\\builds') + else: + return Path('/opt/ccdc/third-party-sources/builds') + + @property + def build_logs(self): + '''Return the directory where build logs are stored''' + if self.windows: + return Path('D:\\tp\\logs') + else: + return Path('/opt/ccdc/third-party-sources/logs') + + @property + def output_base_name(self): + components = [ + self.name, + self.version, + ] + if 'BUILD_BUILDNUMBER' in os.environ: + components.append(os.environ['BUILD_BUILDNUMBER']) + else: + components.append('do-not-use-me-developer-version') + components.append(self.platform) + if self.use_vs_version_in_base_name and 'BUILD_VS_VERSION' in os.environ: + components.append(f'vs{os.environ["BUILD_VS_VERSION"]}') + return '-'.join(components) + + @property + def install_directory(self): + '''Return the canonical installation directory''' + return self.toolbase / self.name / self.output_base_name + + @property + def output_archive_filename(self): + return f'{self.output_base_name}.tar.gz' + + def create_archive(self): + if 'BUILD_ARTIFACTSTAGINGDIRECTORY' in os.environ: + archive_output_directory = Path( + os.environ['BUILD_ARTIFACTSTAGINGDIRECTORY']) + else: + archive_output_directory = self.source_builds_base + print(f'Creating {self.output_archive_filename} in {archive_output_directory}') + command = [ + 'tar', + '-zcf', + f'{ archive_output_directory / self.output_archive_filename }', # the tar filename + f'{ self.install_directory.relative_to(self.toolbase / self.name) }', + ] + try: + # keep the name + version directory in the archive, but not the package name directory + self.system(command, cwd=self.toolbase / self.name) + except subprocess.CalledProcessError as e: + if not self.windows: + raise e + command.insert(1, '--force-local') + # keep the name + version directory in the archive, but not the package name directory + self.system(command, cwd=self.toolbase / self.name) + + @property + def include_directories(self): + '''Return the directories clients must add to their include path''' + return [self.install_directory / 'include'] + + @property + def library_link_directories(self): + '''Return the directories clients must add to their library link path''' + return [self.install_directory / 'lib'] + + @property + def source_archives(self): + '''Map of archive file/url to fetch''' + return {} + + def fetch_source_archives(self): + import urllib.request + for filename, url in self.source_archives.items(): + if (self.source_downloads_base / filename).exists(): + print( + f'Skipping download of existing {self.source_downloads_base / filename}') + continue + print(f'Fetching {url} to {self.source_downloads_base / filename}') + with urllib.request.urlopen(url) as response: + with open(self.source_downloads_base / filename, 'wb') as final_file: + shutil.copyfileobj(response, final_file) + + def extract_source_archives(self): + for source_archive_filename in self.source_archives.keys(): + self.extract_archive(self.source_downloads_base / + source_archive_filename, self.source_extracted) + + def extract_archive(self, path, where): + '''untar a file with any reasonable suffix''' + print(f'Extracting {path} to {where}') + if '.zip' in path.suffixes: + self.system(['unzip', '-q', '-o', str(path)], cwd=where) + return + if '.bz2' in path.suffixes: + flags = 'jxf' + elif '.gz' in path.suffixes: + flags = 'zxf' + elif '.tgz' in path.suffixes: + flags = 'zxf' + elif '.xz' in path.suffixes: + flags = 'xf' + else: + raise AttributeError(f"Can't extract {path}") + + if self.windows: + flags = f'-{flags}' + try: + self.system(['tar', '--force-local', + flags, str(path)], cwd=where) + except subprocess.CalledProcessError: + self.system(['tar', flags, str(path)], cwd=where) + else: + self.system(['tar', flags, str(path)], cwd=where) + + def patch_sources(self): + '''Override to patch source code after extraction''' + pass + + @property + def source_downloads(self): + p = self.source_downloads_base / self.name + p.mkdir(parents=True, exist_ok=True) + return p + + @property + def source_extracted(self): + p = self.source_extracted_base / self.name + p.mkdir(parents=True, exist_ok=True) + return p + + @property + def main_source_directory_path(self): + return self.source_extracted / f'{self.name}-{self.version}' + + @property + def build_directory_path(self): + p = self.source_builds_base / self.name + p.mkdir(parents=True, exist_ok=True) + return p + + def cleanup(self): + try: + shutil.rmtree(self.source_extracted, ignore_errors=True) + print(f'Cleaned up {self.source_extracted}') + except OSError: + pass + try: + shutil.rmtree(self.build_directory_path, ignore_errors=True) + print(f'Cleaned up {self.build_directory_path}') + except OSError: + pass + + @property + def configuration_script(self): + return None + + @property + def arguments_to_configuration_script(self): + return [f'--prefix={self.install_directory}'] + + @property + def cxxflags(self): + flags = [ + '-O2' + ] + if self.macos: + flags.extend([ + '-arch', 'x86_64', + '-isysroot', self.macos_sdkroot, + f'-mmacosx-version-min={self.macos_deployment_target}', + ]) + return flags + + @property + def ldflags(self): + flags = [] + if self.macos: + flags.extend([ + '-arch', 'x86_64', + '-isysroot', self.macos_sdkroot, + f'-mmacosx-version-min={self.macos_deployment_target}', + ]) + return flags + + @property + def cflags(self): + flags = [ + '-O2' + ] + if self.macos: + flags.extend([ + '-arch', 'x86_64', + '-isysroot', self.macos_sdkroot, + f'-mmacosx-version-min={self.macos_deployment_target}', + ]) + return flags + + @property + def environment_for_configuration_script(self): + env = dict(os.environ) + if self.cflags: + env['CFLAGS'] = ' '.join(self.cflags) + if self.cxxflags: + env['CXXFLAGS'] = ' '.join(self.cxxflags) + if self.ldflags: + env['LDFLAGS'] = ' '.join(self.ldflags) + if self.macos: + env['MACOSX_DEPLOYMENT_TARGET'] = self.macos_deployment_target + return env + + def run_configuration_script(self): + '''run the required commands to configure a package''' + if not self.configuration_script: + print(f'Skipping configuration script for {self.name}') + return + st = os.stat(self.configuration_script) + if '/usr/' not in str(self.configuration_script): + os.chmod(self.configuration_script, st.st_mode | stat.S_IEXEC) + self.system( + [str(self.configuration_script)] + + self.arguments_to_configuration_script, + env=self.environment_for_configuration_script, cwd=self.build_directory_path) + + @property + def environment_for_build_command(self): + return self.environment_for_configuration_script + + def run_build_command(self): + '''run the required commands to build a package after configuration''' + pass + + def run_install_command(self): + '''run the required commands to install a package''' + pass + + def logfile_path(self, task): + '''Canonical log file for a particular task''' + return self.build_logs / f'{self.name}-{self.version}-{task}.log' + + def system(self, command, cwd=None, env=None, append_log=False): + '''execute command, logging in the appropriate logfile''' + task = sys._getframe(1).f_code.co_name + print(f'{self.name} {task}') + if isinstance(command, str): + command = [command] + print(f'Running {command}') + openmode = 'a' if append_log else 'w' + with open(self.logfile_path(task), openmode) as f: + output = '' + p = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=env) + while True: + retcode = p.poll() + l = p.stdout.readline().decode('utf-8') + print(l.rstrip()) + output += l + f.write(l) + if retcode is not None: + break + assert p.returncode is not None + if p.returncode != 0: + print(f'Failed process environment was {env}') + raise subprocess.CalledProcessError( + returncode=p.returncode, cmd=command, output=output) + + def verify(self): + '''Override this function to verify that the install has + produced something functional.''' + pass + + def build(self): + self.cleanup() + self.fetch_source_archives() + self.extract_source_archives() + self.patch_sources() + self.run_configuration_script() + self.run_build_command() + self.run_install_command() + self.verify() + self.create_archive() + + def update_dylib_id(self, library_path, new_id): + '''MacOS helper to change a library's identifier''' + self.system(['install_name_tool', '-id', new_id, str(library_path)]) + + def change_dylib_lookup(self, library_path, from_path, to_path): + '''MacOS helper to change the path where libraries and executables look for other libraries''' + self.system(['install_name_tool', '-change', + from_path, to_path, str(library_path)]) + + def patch(self, fname, *subs): + with open(fname) as read_file: + txt = read_file.read() + for (old, new) in subs: + txt = txt.replace(old, new) + with open(fname, 'w') as out: + out.write(txt) + + +_pkg = Package() +_pkg.prepare_directories() +if _pkg.macos: + assert os.path.exists(_pkg.macos_sdkroot) +_pkg = None + + +class GnuMakeMixin(object): + '''Make based build''' + + def run_build_command(self): + self.system(['make', f'-j{multiprocessing.cpu_count()}'], + env=self.environment_for_build_command, cwd=self.build_directory_path) + + +class MakeInstallMixin(object): + '''Make install (rather than the default do nothing install)''' + + def run_install_command(self): + self.system(['make', 'install'], + env=self.environment_for_build_command, cwd=self.build_directory_path) + + +class AutoconfMixin(GnuMakeMixin, MakeInstallMixin, object): + '''Autoconf based configure script''' + @property + def configuration_script(self): + return self.main_source_directory_path / 'configure' + + +class CMakeMixin(Package): + @property + def configuration_script(self): + return shutil.which('cmake') + + def run_build_command(self): + self.system([self.configuration_script, '--build', '.', '--config', 'Release'], + env=self.environment_for_build_command, cwd=self.build_directory_path) + + def run_install_command(self): + self.system([self.configuration_script, '--install', '.'], + env=self.environment_for_build_command, cwd=self.build_directory_path) + + @property + def visual_studio_generator_for_build(self): + if not "BUILD_VS_VERSION" in os.environ: + print('BUILD_VS_VERSION not set, defaulting to VS 2019') + return 'Visual Studio 16 2019' + if os.environ["BUILD_VS_VERSION"] == '2019': + return 'Visual Studio 16 2019' + if os.environ["BUILD_VS_VERSION"] == '2017': + return 'Visual Studio 15 2017' + raise Exception(f'Invalid value for BUILD_VS_VERSION: {os.environ["BUILD_VS_VERSION"]}') + + +class NoArchiveMixin(Package): + def create_archive(self): + pass diff --git a/smoke_test.py b/smoke_test.py index e3ac912..8d38e60 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2,6 +2,10 @@ # there should be no issues importing sqlite libraries import sqlite3 +import distutils.version +# Ensure we haven't inadvertently got the (ancient) system SQLite +assert distutils.version.LooseVersion('3.17.0') <= distutils.version.LooseVersion(sqlite3.sqlite_version) +sqlite3.connect(":memory:") # pyenv has trouble building the tkinter extension, so checking for that # this tends to happen on macos, where even the exception handling below From 1df38bf164ce23bb132173bbca12e2837600b8ba Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 26 Apr 2021 11:10:51 +0100 Subject: [PATCH 22/26] chown destdir --- build_python.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_python.py b/build_python.py index 00394e4..b2ccfc7 100755 --- a/build_python.py +++ b/build_python.py @@ -120,8 +120,9 @@ def python_interpreter(): def prepare_output_dir(): if linux(): + subprocess.run(f'sudo mkdir -p {python_destdir()}', shell=True) - subprocess.run(f'sudo chown $USER {python_destdir()}', shell=True) + subprocess.run(f'sudo chown $USER {python_destdir()}; echo "chown $USER {python_destdir()}"', shell=True) def install_from_msi(): import urllib.request @@ -145,6 +146,8 @@ def install_prerequisites(): subprocess.run('sudo yum update -y', shell=True, check=True) subprocess.run('sudo yum install -y findutils gcc zlib-devel bzip2 bzip2-devel readline-devel openssl-devel tk-devel xz xz-devel libffi-devel', shell=True, check=True) # See https://jira.ccdc.cam.ac.uk/browse/BLD-5684 + subprocess.run(f'sudo mkdir -p {python_version_destdir()}', shell=True) + subprocess.run(f'sudo chown $USER {python_version_destdir()}; echo "chown $USER {python_version_destdir()}"', shell=True) SqlitePackage().build() if ubuntu(): subprocess.run('sudo apt-get -y update', shell=True, check=True) From 32688035ae49093036a559ee07eb76fae2856153 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Mon, 26 Apr 2021 11:33:13 +0100 Subject: [PATCH 23/26] use id-u to find user --- build_python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_python.py b/build_python.py index b2ccfc7..02b0712 100755 --- a/build_python.py +++ b/build_python.py @@ -122,7 +122,7 @@ def prepare_output_dir(): if linux(): subprocess.run(f'sudo mkdir -p {python_destdir()}', shell=True) - subprocess.run(f'sudo chown $USER {python_destdir()}; echo "chown $USER {python_destdir()}"', shell=True) + subprocess.run(f'sudo chown $(id -u) {python_destdir()}; echo "chown $(id -u) {python_destdir()}"', shell=True) def install_from_msi(): import urllib.request @@ -147,7 +147,7 @@ def install_prerequisites(): subprocess.run('sudo yum install -y findutils gcc zlib-devel bzip2 bzip2-devel readline-devel openssl-devel tk-devel xz xz-devel libffi-devel', shell=True, check=True) # See https://jira.ccdc.cam.ac.uk/browse/BLD-5684 subprocess.run(f'sudo mkdir -p {python_version_destdir()}', shell=True) - subprocess.run(f'sudo chown $USER {python_version_destdir()}; echo "chown $USER {python_version_destdir()}"', shell=True) + subprocess.run(f'sudo chown $(id -u) {python_version_destdir()}; echo "chown $(id -u) {python_version_destdir()}"', shell=True) SqlitePackage().build() if ubuntu(): subprocess.run('sudo apt-get -y update', shell=True, check=True) From 67ea466b02b2820e3a34e4bf3aa71de37667afe5 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Sun, 3 Apr 2022 17:31:55 +0100 Subject: [PATCH 24/26] Update base python versions NO_JIRA --- build_python.py | 35 +++++++++++++++++++++++++---------- common-tasks.yml | 16 ++++++++-------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/build_python.py b/build_python.py index 02b0712..82695e2 100755 --- a/build_python.py +++ b/build_python.py @@ -3,6 +3,8 @@ import subprocess import sys import os +from platform import processor + from pathlib import Path from ccdc.thirdparty.package import Package, AutoconfMixin, MakeInstallMixin, NoArchiveMixin, CMakeMixin @@ -18,13 +20,13 @@ def install_directory(self): class SqlitePackage(InstallInBasePythonMixin, AutoconfMixin, NoArchiveMixin, Package): '''SQLite''' name = 'sqlite' - version = '3.35.5' - tarversion = '3350500' + version = '3.38.2' + tarversion = '3380200' @property def source_archives(self): return { - f'sqlite-autoconf-{self.tarversion}.tar.gz': f'https://www.sqlite.org/2021/sqlite-autoconf-{self.tarversion}.tar.gz' + f'sqlite-autoconf-{self.tarversion}.tar.gz': f'https://www.sqlite.org/2022/sqlite-autoconf-{self.tarversion}.tar.gz' } @property @@ -89,6 +91,10 @@ def platform(): else: version = subprocess.check_output('lsb_release -r -s', shell=True).decode('utf-8').strip() return f'ubuntu{version}' + if macos(): + if processor() == 'arm': + return 'darwin-arm' + return sys.platform def output_base_name(): @@ -164,13 +170,22 @@ def install_pyenv(): def install_pyenv_version(version): python_build_env = dict(os.environ) if macos(): - python_build_env['MACOSX_DEPLOYMENT_TARGET']=f'{macos_deployment_target}' - python_build_env['PATH']=f"/usr/local/opt/tcl-tk/bin:{python_build_env['PATH']}" - python_build_env['LDFLAGS']=f"-L/usr/local/opt/tcl-tk/lib -mmacosx-version-min={macos_deployment_target}" - python_build_env['CFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" - python_build_env['CPPFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" - python_build_env['PKG_CONFIG_PATH']="/usr/local/opt/tcl-tk/lib/pkgconfig" - python_build_env['PYTHON_CONFIGURE_OPTS']="--with-tcltk-includes='-I/usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6' --enable-shared" + if processor() == 'arm': + python_build_env['MACOSX_DEPLOYMENT_TARGET']=f'{macos_deployment_target}' + python_build_env['PATH']=f"/opt/homebrew//opt/tcl-tk/bin:{python_build_env['PATH']}" + python_build_env['LDFLAGS']=f"-L/opt/homebrew/opt/tcl-tk/lib -mmacosx-version-min={macos_deployment_target}" + python_build_env['CFLAGS']=f"-I/opt/homebrew/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" + python_build_env['CPPFLAGS']=f"-I/opt/homebrew/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" + python_build_env['PKG_CONFIG_PATH']="/opt/homebrew/opt/tcl-tk/lib/pkgconfig" + python_build_env['PYTHON_CONFIGURE_OPTS']="--with-tcltk-includes='-I/opt/homebrew/opt/tcl-tk/include' --with-tcltk-libs='-L/opt/homebrew/opt/tcl-tk/lib -ltcl8.6 -ltk8.6' --enable-shared" + else: + python_build_env['MACOSX_DEPLOYMENT_TARGET']=f'{macos_deployment_target}' + python_build_env['PATH']=f"/usr/local/opt/tcl-tk/bin:{python_build_env['PATH']}" + python_build_env['LDFLAGS']=f"-L/usr/local/opt/tcl-tk/lib -mmacosx-version-min={macos_deployment_target}" + python_build_env['CFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" + python_build_env['CPPFLAGS']=f"-I/usr/local/opt/tcl-tk/include -mmacosx-version-min={macos_deployment_target}" + python_build_env['PKG_CONFIG_PATH']="/usr/local/opt/tcl-tk/lib/pkgconfig" + python_build_env['PYTHON_CONFIGURE_OPTS']="--with-tcltk-includes='-I/usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6' --enable-shared" subprocess.run(f'sudo "--preserve-env=MACOSX_DEPLOYMENT_TARGET,PATH,LDFLAGS,CFLAGS,CPPFLAGS,PKG_CONFIG_PATH,PYTHON_CONFIGURE_OPTS" python-build -v {version} {python_version_destdir()}', shell=True, check=True, env=python_build_env) diff --git a/common-tasks.yml b/common-tasks.yml index 12aa6e8..20b1de8 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -4,22 +4,22 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.6.8 #later versions don't have windows installers - displayName: 'install python 3.6.8' + arguments: 3.7.9 # later versions don't have windows installers + displayName: 'install python 3.7.9' - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.7.9 #later versions don't have windows installers - displayName: 'install python 3.7.9' + arguments: 3.9.11 + displayName: 'install python 3.9.11' - task: PythonScript@0 inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.9.4 # added in pyenv v1.2.26 - displayName: 'install python 3.9.1' + arguments: 3.10.3 + displayName: 'install python 3.10.3' # Building 3.8 last on purpose because the installer on windows clobbers the system's hosted python # Just keep it last or wonder why the system is loading python2 afterwards... @@ -27,8 +27,8 @@ steps: inputs: scriptSource: 'filePath' scriptPath: build_python.py - arguments: 3.8.9 # added in pyenv v1.2.25 - displayName: 'install python 3.8.9' + arguments: 3.8.10 # later versions don't have windows installers + displayName: 'install python 3.8.10' # Upload artifactory build info - task: ArtifactoryCollectIssues@1 From fa8c14a27aefdbc002d758824ae9b311f5cc4faf Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Sun, 3 Apr 2022 19:00:44 +0100 Subject: [PATCH 25/26] try condition --- common-tasks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common-tasks.yml b/common-tasks.yml index 20b1de8..dceb84a 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -15,6 +15,8 @@ steps: displayName: 'install python 3.9.11' - task: PythonScript@0 + # requires more up to date openssl than one provided by centos 7 + condition: ne(variables.artifactory.platformName, 'centos7') inputs: scriptSource: 'filePath' scriptPath: build_python.py From d51045bd506792c93688c192f88aaa625d0c8f54 Mon Sep 17 00:00:00 2001 From: Claudio Bantaloukas Date: Sun, 3 Apr 2022 19:34:39 +0100 Subject: [PATCH 26/26] try checking condition differently --- .gitignore | 1 + common-tasks.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/common-tasks.yml b/common-tasks.yml index dceb84a..562b4df 100644 --- a/common-tasks.yml +++ b/common-tasks.yml @@ -16,7 +16,7 @@ steps: - task: PythonScript@0 # requires more up to date openssl than one provided by centos 7 - condition: ne(variables.artifactory.platformName, 'centos7') + condition: ne('$(artifactory.platformName)', 'centos7') inputs: scriptSource: 'filePath' scriptPath: build_python.py