From e269006eb1eeef9cb4d5c1f14df3fa14b88f9024 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 17 Jun 2019 17:37:55 +0200 Subject: [PATCH 1/4] update to knox 1.2 and fix files permission for user knox --- knox-csd/build.sh | 2 +- knox-parcel/defaults/main.yml | 9 +-- knox-parcel/files/make_manifest.py | 118 +++++++++++++++++++++++++++++ knox-parcel/tasks/main.yml | 38 ++++++---- knox-parcel/vars/main.yml | 2 +- 5 files changed, 147 insertions(+), 22 deletions(-) create mode 100644 knox-parcel/files/make_manifest.py diff --git a/knox-csd/build.sh b/knox-csd/build.sh index 340b5aa..429820b 100755 --- a/knox-csd/build.sh +++ b/knox-csd/build.sh @@ -1,3 +1,3 @@ #!/bin/bash -jar -cvf KNOX-1.0.jar * +jar -cvf KNOX-1.2.jar * diff --git a/knox-parcel/defaults/main.yml b/knox-parcel/defaults/main.yml index ccae7e2..0c9e84e 100644 --- a/knox-parcel/defaults/main.yml +++ b/knox-parcel/defaults/main.yml @@ -11,13 +11,8 @@ BASE_DIR: "{{ lookup('env','PWD') }}/work" WORK_DIR: "{{ BASE_DIR | regex_replace('\\/$', '') }}/{{ WORK_DIRECTORY_NAME | regex_replace('\\/$', '') }}" OUTPUT_DIR: "{{ BASE_DIR }}/{{ OUTPUT_DIRECTORY_NAME | regex_replace('\\/$', '') }}" CACHE_DIR: "{{ BASE_DIR }}/{{ CACHE_DIRECTORY_NAME }}" - -TARGET_ARCHITECTURES: - - el6 - - el7 - - sles12 - - xenial +MANIFEST_DIR: "{{ BASE_DIR }}/knox-manifest" PROJECT_NAME_LOWER: "{{ PROJECT_NAME | lower }}" PROJECT_NAME_UPPER: "{{ PROJECT_NAME | upper }}" -PARCEL_PREFIX: "{{ PROJECT_NAME_UPPER }}-{{ PROJECT_VERSION }}-{{ PARCEL_VERSION }}" \ No newline at end of file +PARCEL_PREFIX: "{{ PROJECT_NAME_UPPER }}-{{ PROJECT_VERSION }}-{{ PARCEL_VERSION }}" diff --git a/knox-parcel/files/make_manifest.py b/knox-parcel/files/make_manifest.py new file mode 100644 index 0000000..dab56db --- /dev/null +++ b/knox-parcel/files/make_manifest.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This program creates a manifest.json file from a directory of parcels and +# places the file in the same directory as the parcels. +# Once created, the directory can be served over http as a parcel repository. + +import hashlib +import json +import os +import re +import sys +import tarfile +import time + +def _get_parcel_dirname(parcel_name): + """ + Extract the required parcel directory name for a given parcel. + + eg: CDH-5.0.0-el6.parcel -> CDH-5.0.0 + """ + parts = re.match(r"^(.*?)-(.*)-(.*?)$", parcel_name).groups() + return parts[0] + '-' + parts[1] + +def _safe_copy(key, src, dest): + """ + Conditionally copy a key/value pair from one dictionary to another. + + Nothing is done if the key is not present in the source dictionary + """ + if key in src: + dest[key] = src[key] + +def make_manifest(path, timestamp=time.time()): + """ + Make a manifest.json document from the contents of a directory. + + This function will scan the specified directory, identify any parcel files + in it, and then build a manifest from those files. Certain metadata will be + extracted from the parcel and copied into the manifest. + + @param path: The path of the directory to scan for parcels + @param timestamp: Unix timestamp to place in manifest.json + @return: the manifest.json as a string + """ + manifest = {} + manifest['lastUpdated'] = int(timestamp * 1000) + manifest['parcels'] = [] + + files = os.listdir(path) + for f in files: + if not f.endswith('.parcel'): + continue + + print("Found parcel %s" % (f,)) + entry = {} + entry['parcelName'] = f + + fullpath = os.path.join(path, f) + + with open(fullpath, 'rb') as fp: + entry['hash'] = hashlib.sha1(fp.read()).hexdigest() + + with tarfile.open(fullpath, 'r') as tar: + try: + json_member = tar.getmember(os.path.join(_get_parcel_dirname(f), + 'meta', 'parcel.json')) + except KeyError: + print("Parcel does not contain parcel.json") + continue + try: + parcel = json.loads(tar.extractfile(json_member).read().decode(encoding='UTF-8')) + except: + print("Failed to parse parcel.json") + continue + _safe_copy('depends', parcel, entry) + _safe_copy('replaces', parcel, entry) + _safe_copy('conflicts', parcel, entry) + _safe_copy('components', parcel, entry) + _safe_copy('servicesRestartInfo', parcel, entry) + + try: + notes_member = tar.getmember(os.path.join(_get_parcel_dirname(f), + 'meta', 'release-notes.txt')) + entry['releaseNotes'] = tar.extractfile(notes_member).read().decode(encoding='UTF-8') + except KeyError: + # No problem if there's no release notes + pass + + manifest['parcels'].append(entry) + + return json.dumps(manifest, indent=4, separators=(',', ': ')) + +if __name__ == "__main__": + path = os.path.curdir + if len(sys.argv) > 1: + path = sys.argv[1] + print("Scanning directory: %s" % (path)) + + manifest = make_manifest(path) + with open(os.path.join(path, 'manifest.json'), 'w') as fp: + fp.write(manifest) + diff --git a/knox-parcel/tasks/main.yml b/knox-parcel/tasks/main.yml index 8d606ae..a947245 100644 --- a/knox-parcel/tasks/main.yml +++ b/knox-parcel/tasks/main.yml @@ -16,6 +16,7 @@ - "{{ OUTPUT_DIR }}" - "{{ WORK_DIR }}" - "{{ CACHE_DIR }}" + - "{{ MANIFEST_DIR }}" - name: Download distribution get_url: @@ -51,6 +52,12 @@ dest: "{{ WORK_DIR }}/{{ PARCEL_PREFIX }}/bin/" mode: 0755 +- name: Copy manifest.py file + copy: + src: make_manifest.py + dest: "{{ MANIFEST_DIR }}/make_manifest.py" + mode: 0755 + - name: Copy modified gateway.sh script copy: src: gateway.sh @@ -66,6 +73,12 @@ mode: u=rwX,g=rX,o=rX recurse: yes +- name: Original directories should be accessible by user knox + file: + path: "{{ WORK_DIR }}/{{ PARCEL_PREFIX }}" + mode: u=rwX,g=rX,o=rX + recurse: yes + - name: Create empty conf directory file: path: "{{ WORK_DIR }}/{{ PARCEL_PREFIX }}/conf" @@ -79,18 +92,17 @@ - name: Create parcel archive: path: "{{ WORK_DIR }}/" - dest: "{{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-universal.parcel" + dest: "{{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-el7.parcel" -- name: Symlink parcel to architecture specific parcels - file: - src: "{{ PARCEL_PREFIX }}-universal.parcel" - path: "{{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-{{ item }}.parcel" - state: link - with_items: "{{ TARGET_ARCHITECTURES }}" +- name: Create manifest + shell: python {{ MANIFEST_DIR }}/make_manifest.py {{ OUTPUT_DIR }} -- name: Empty working directory - file: - path: "{{ item }}" - state: absent - with_items: - - "{{ WORK_DIR }}" \ No newline at end of file +- name: Create checksum file + shell: sha1sum {{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-el7.parcel > {{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-el7.parcel.sha1 + +#- name: Empty working directory +# file: +# path: "{{ item }}" +# state: absent +# with_items: +# - "{{ WORK_DIR }}" diff --git a/knox-parcel/vars/main.yml b/knox-parcel/vars/main.yml index 4c6ea57..7dde956 100644 --- a/knox-parcel/vars/main.yml +++ b/knox-parcel/vars/main.yml @@ -1,6 +1,6 @@ --- PROJECT_NAME: knox -PROJECT_VERSION: 1.1.0 +PROJECT_VERSION: 1.2.0 PARCEL_VERSION: 1 DOWNLOAD_FILE_NAME: knox-{{ PROJECT_VERSION }}.zip From 2d8f7fde70b0ad827c9c13ce62ac1c75be2ab9e1 Mon Sep 17 00:00:00 2001 From: AliGouta Date: Mon, 17 Jun 2019 17:37:55 +0200 Subject: [PATCH 2/4] update to knox 1.2 and fix files permission for user knox --- knox-csd/build.sh | 2 +- knox-parcel/defaults/main.yml | 9 +-- knox-parcel/files/make_manifest.py | 118 +++++++++++++++++++++++++++++ knox-parcel/tasks/main.yml | 38 ++++++---- knox-parcel/vars/main.yml | 2 +- 5 files changed, 147 insertions(+), 22 deletions(-) create mode 100644 knox-parcel/files/make_manifest.py diff --git a/knox-csd/build.sh b/knox-csd/build.sh index 340b5aa..429820b 100755 --- a/knox-csd/build.sh +++ b/knox-csd/build.sh @@ -1,3 +1,3 @@ #!/bin/bash -jar -cvf KNOX-1.0.jar * +jar -cvf KNOX-1.2.jar * diff --git a/knox-parcel/defaults/main.yml b/knox-parcel/defaults/main.yml index ccae7e2..0c9e84e 100644 --- a/knox-parcel/defaults/main.yml +++ b/knox-parcel/defaults/main.yml @@ -11,13 +11,8 @@ BASE_DIR: "{{ lookup('env','PWD') }}/work" WORK_DIR: "{{ BASE_DIR | regex_replace('\\/$', '') }}/{{ WORK_DIRECTORY_NAME | regex_replace('\\/$', '') }}" OUTPUT_DIR: "{{ BASE_DIR }}/{{ OUTPUT_DIRECTORY_NAME | regex_replace('\\/$', '') }}" CACHE_DIR: "{{ BASE_DIR }}/{{ CACHE_DIRECTORY_NAME }}" - -TARGET_ARCHITECTURES: - - el6 - - el7 - - sles12 - - xenial +MANIFEST_DIR: "{{ BASE_DIR }}/knox-manifest" PROJECT_NAME_LOWER: "{{ PROJECT_NAME | lower }}" PROJECT_NAME_UPPER: "{{ PROJECT_NAME | upper }}" -PARCEL_PREFIX: "{{ PROJECT_NAME_UPPER }}-{{ PROJECT_VERSION }}-{{ PARCEL_VERSION }}" \ No newline at end of file +PARCEL_PREFIX: "{{ PROJECT_NAME_UPPER }}-{{ PROJECT_VERSION }}-{{ PARCEL_VERSION }}" diff --git a/knox-parcel/files/make_manifest.py b/knox-parcel/files/make_manifest.py new file mode 100644 index 0000000..dab56db --- /dev/null +++ b/knox-parcel/files/make_manifest.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This program creates a manifest.json file from a directory of parcels and +# places the file in the same directory as the parcels. +# Once created, the directory can be served over http as a parcel repository. + +import hashlib +import json +import os +import re +import sys +import tarfile +import time + +def _get_parcel_dirname(parcel_name): + """ + Extract the required parcel directory name for a given parcel. + + eg: CDH-5.0.0-el6.parcel -> CDH-5.0.0 + """ + parts = re.match(r"^(.*?)-(.*)-(.*?)$", parcel_name).groups() + return parts[0] + '-' + parts[1] + +def _safe_copy(key, src, dest): + """ + Conditionally copy a key/value pair from one dictionary to another. + + Nothing is done if the key is not present in the source dictionary + """ + if key in src: + dest[key] = src[key] + +def make_manifest(path, timestamp=time.time()): + """ + Make a manifest.json document from the contents of a directory. + + This function will scan the specified directory, identify any parcel files + in it, and then build a manifest from those files. Certain metadata will be + extracted from the parcel and copied into the manifest. + + @param path: The path of the directory to scan for parcels + @param timestamp: Unix timestamp to place in manifest.json + @return: the manifest.json as a string + """ + manifest = {} + manifest['lastUpdated'] = int(timestamp * 1000) + manifest['parcels'] = [] + + files = os.listdir(path) + for f in files: + if not f.endswith('.parcel'): + continue + + print("Found parcel %s" % (f,)) + entry = {} + entry['parcelName'] = f + + fullpath = os.path.join(path, f) + + with open(fullpath, 'rb') as fp: + entry['hash'] = hashlib.sha1(fp.read()).hexdigest() + + with tarfile.open(fullpath, 'r') as tar: + try: + json_member = tar.getmember(os.path.join(_get_parcel_dirname(f), + 'meta', 'parcel.json')) + except KeyError: + print("Parcel does not contain parcel.json") + continue + try: + parcel = json.loads(tar.extractfile(json_member).read().decode(encoding='UTF-8')) + except: + print("Failed to parse parcel.json") + continue + _safe_copy('depends', parcel, entry) + _safe_copy('replaces', parcel, entry) + _safe_copy('conflicts', parcel, entry) + _safe_copy('components', parcel, entry) + _safe_copy('servicesRestartInfo', parcel, entry) + + try: + notes_member = tar.getmember(os.path.join(_get_parcel_dirname(f), + 'meta', 'release-notes.txt')) + entry['releaseNotes'] = tar.extractfile(notes_member).read().decode(encoding='UTF-8') + except KeyError: + # No problem if there's no release notes + pass + + manifest['parcels'].append(entry) + + return json.dumps(manifest, indent=4, separators=(',', ': ')) + +if __name__ == "__main__": + path = os.path.curdir + if len(sys.argv) > 1: + path = sys.argv[1] + print("Scanning directory: %s" % (path)) + + manifest = make_manifest(path) + with open(os.path.join(path, 'manifest.json'), 'w') as fp: + fp.write(manifest) + diff --git a/knox-parcel/tasks/main.yml b/knox-parcel/tasks/main.yml index 8d606ae..a947245 100644 --- a/knox-parcel/tasks/main.yml +++ b/knox-parcel/tasks/main.yml @@ -16,6 +16,7 @@ - "{{ OUTPUT_DIR }}" - "{{ WORK_DIR }}" - "{{ CACHE_DIR }}" + - "{{ MANIFEST_DIR }}" - name: Download distribution get_url: @@ -51,6 +52,12 @@ dest: "{{ WORK_DIR }}/{{ PARCEL_PREFIX }}/bin/" mode: 0755 +- name: Copy manifest.py file + copy: + src: make_manifest.py + dest: "{{ MANIFEST_DIR }}/make_manifest.py" + mode: 0755 + - name: Copy modified gateway.sh script copy: src: gateway.sh @@ -66,6 +73,12 @@ mode: u=rwX,g=rX,o=rX recurse: yes +- name: Original directories should be accessible by user knox + file: + path: "{{ WORK_DIR }}/{{ PARCEL_PREFIX }}" + mode: u=rwX,g=rX,o=rX + recurse: yes + - name: Create empty conf directory file: path: "{{ WORK_DIR }}/{{ PARCEL_PREFIX }}/conf" @@ -79,18 +92,17 @@ - name: Create parcel archive: path: "{{ WORK_DIR }}/" - dest: "{{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-universal.parcel" + dest: "{{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-el7.parcel" -- name: Symlink parcel to architecture specific parcels - file: - src: "{{ PARCEL_PREFIX }}-universal.parcel" - path: "{{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-{{ item }}.parcel" - state: link - with_items: "{{ TARGET_ARCHITECTURES }}" +- name: Create manifest + shell: python {{ MANIFEST_DIR }}/make_manifest.py {{ OUTPUT_DIR }} -- name: Empty working directory - file: - path: "{{ item }}" - state: absent - with_items: - - "{{ WORK_DIR }}" \ No newline at end of file +- name: Create checksum file + shell: sha1sum {{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-el7.parcel > {{ OUTPUT_DIR }}/{{ PARCEL_PREFIX }}-el7.parcel.sha1 + +#- name: Empty working directory +# file: +# path: "{{ item }}" +# state: absent +# with_items: +# - "{{ WORK_DIR }}" diff --git a/knox-parcel/vars/main.yml b/knox-parcel/vars/main.yml index 4c6ea57..7dde956 100644 --- a/knox-parcel/vars/main.yml +++ b/knox-parcel/vars/main.yml @@ -1,6 +1,6 @@ --- PROJECT_NAME: knox -PROJECT_VERSION: 1.1.0 +PROJECT_VERSION: 1.2.0 PARCEL_VERSION: 1 DOWNLOAD_FILE_NAME: knox-{{ PROJECT_VERSION }}.zip From d60e528a2cb65a8bd1ce927157bf252fd7e75505 Mon Sep 17 00:00:00 2001 From: AliGouta Date: Mon, 17 Jun 2019 17:56:37 +0200 Subject: [PATCH 3/4] Update README.adoc --- README.adoc | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/README.adoc b/README.adoc index c885701..0940452 100644 --- a/README.adoc +++ b/README.adoc @@ -1,6 +1,6 @@ = Apache Knox Parcel & CSD -This repository contains an Ansible playbook to create a Parcel for Apache Knox as well as the definitions needed to build a CSD. +This repository contains an Ansible playbook to create a Parcel for Apache Knox 1.2 as well as the definitions needed to build a CSD. Both these things are needed if you want to integrate Knox into Cloudera Manager. == Parcel @@ -9,17 +9,8 @@ To build the Parcel execute the `parcel.yml` playbook from the main directory us There is a script called `build-parcel.sh` that can be used to kick off the process. The finished parcel will be in the `work/output` directory. Cloudera Manager requires one Parcel per supported target distribution. -We only create a single physical file (with the suffix `-universal`) and symlink the others. -Note: This does not include the call to `make_manifest.py` which is needed to create the `manifest.json` file. - -You can use this Parcel and drop it in the `/opt/cloudera/parcel-cache` directory. -For RedHat/CentOS you need to change the suffix to `-el7`. -For a full list of suffixes check the https://github.com/cloudera/cm_ext/wiki/Parcel-distro-suffixes[cm-ext Github wiki]. -If you go this route you also need to create a `*.sha` file containing the SHA checksum. -This can be calculated using `sha1sum`. - -We (http://www.opencore.com[OpenCore]) do not currently host a repository with pre-built Parcels but we might do so in the future. +This only genereates a parcel for el7 distribution including the .sha1 and the manifest == CSD @@ -32,7 +23,7 @@ Copy the resulting JAR file to `/opt/cloudera/csd` and restart Cloudera Manager. You also need to restart the _Cloudera Management Service_ once! To configure Knox you'll need to _manually_ change the topologies etc. in the data directory itself. -The https://knox.apache.org/books/knox-1-1-0/user-guide.html[Knox User Guide] can help. +The https://knox.apache.org/books/knox-1-2-0/user-guide.html[Knox User Guide] can help. You will also need to manually create Proxy Users in the various services that Knox should access. @@ -63,4 +54,4 @@ The only way to have the environment variable take effect is by removing the def The directory `conf` needs to stay though because `gateway.sh` checks for its existence. The path is hardcoded in the script and cannot be changed even though its pointing to the wrong location. Solution is to create an empty `conf` directory or to patch the `gateway.sh` file. -This parcel does the former. \ No newline at end of file +This parcel does the former. From 9f8c63c19d385a5ca57b58e0fc3b57b4aaf17560 Mon Sep 17 00:00:00 2001 From: AliGouta Date: Sat, 22 Jun 2019 01:02:31 +0200 Subject: [PATCH 4/4] fix data directory and sdl file --- knox-csd/build.sh | 2 +- knox-csd/descriptor/service.sdl | 2 +- knox-csd/scripts/bootstrap_config.sh | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/knox-csd/build.sh b/knox-csd/build.sh index 429820b..6de5831 100755 --- a/knox-csd/build.sh +++ b/knox-csd/build.sh @@ -1,3 +1,3 @@ #!/bin/bash -jar -cvf KNOX-1.2.jar * +jar -cvf KNOX-1.2.0.jar * diff --git a/knox-csd/descriptor/service.sdl b/knox-csd/descriptor/service.sdl index 70dba82..e12efbf 100644 --- a/knox-csd/descriptor/service.sdl +++ b/knox-csd/descriptor/service.sdl @@ -221,7 +221,7 @@ "requiredRoleState": "stopped", "commandRunner": { "program": "scripts/bootstrap_config.sh", - "args": ["${conf_dir}"] + "args": ["${conf_dir}", "${data_dir}"] } }, { diff --git a/knox-csd/scripts/bootstrap_config.sh b/knox-csd/scripts/bootstrap_config.sh index 78aa02f..2eef2e0 100644 --- a/knox-csd/scripts/bootstrap_config.sh +++ b/knox-csd/scripts/bootstrap_config.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash GATEWAY_CONF_HOME="$1" +GATEWAY_DATA_HOME="$2" echo "cp -R $KNOX_HOME/conf.org/* $GATEWAY_CONF_HOME/" -cp -R "$KNOX_HOME/conf.org/"* "$GATEWAY_CONF_HOME/" \ No newline at end of file +cp -R "$KNOX_HOME/conf.org/"* "$GATEWAY_CONF_HOME/" + +echo "cp -R $KNOX_HOME/data/* $GATEWAY_DATA_HOME/" +cp -R "$KNOX_HOME/data/"* "$GATEWAY_DATA_HOME/"