Change Gateway offline install method to use pip wheel/download (#287)

* Change offline install method to use pip wheel/download
* Update docs
* Drop support for old Gateway versions
* Remove separate Gateway Ansible install tasks since Ansible is installed automatically when installing the Gateway wheel

Author: kvelarde-itential

README.md

@@ -573,8 +573,8 @@ all:
       hosts:
         example2.host.com:
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.9-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: automation_gateway-4.3.56-py3-none-any.whl
 ```
 
 ### Run the Itential Deployer
@@ -735,8 +735,8 @@ all:
       hosts:
         example2.host.com:
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.9-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: automation_gateway-4.3.56-py3-none-any.whl
 ```
 
 ### Minimal Architecture Inventory
@@ -775,8 +775,8 @@ all:
       hosts:
         automation-gateway.host.com:
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.9-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: automation_gateway-4.3.56-py3-none-any.whl
 ```
 
 ### Highly Available Architecture Inventory
@@ -829,8 +829,8 @@ all:
       hosts:
         automation-gateway1.host.com:
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.9-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: automation_gateway-4.3.56-py3-none-any.whl
 ```
 
 ### Highly Available Architecture Inventory leveraging external dependencies
@@ -871,8 +871,8 @@ all:
       hosts:
         automation-gateway1.host.com:
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.9-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: automation_gateway-4.3.56-py3-none-any.whl
 ```
 
 ### Active/Standby Architecture Inventory
@@ -964,8 +964,8 @@ all:
       hosts:
         datacenter2.automation-gateway1.host.com:
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.9-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: automation_gateway-4.3.56-py3-none-any.whl
 ```
 
 ## Component Guides

docs/itential_gateway_guide.md

@@ -134,7 +134,7 @@ all:
         <host1>:
           ansible_host: <addr1>
       vars:
-        gateway_release: 2023.1
+        gateway_release: 4.3
         gateway_whl_file: <wheel-file>
 ```
 
@@ -151,7 +151,7 @@ all:
         <host1>:
           ansible_host: <addr1>
       vars:
-        gateway_release: 2023.1
+        gateway_release: 4.3
         gateway_whl_file: <wheel-file>
         gateway_https: true
 ```
@@ -169,7 +169,7 @@ all:
         <host1>:
           ansible_host: <addr1>
       vars:
-        gateway_release: 2023.1
+        gateway_release: 4.3
         gateway_whl_file: <wheel-file>
         gateway_haproxy_enabled: true
 ```

docs/patch_itential_gateway_guide.md

@@ -33,8 +33,8 @@ all:
         <host1:
           ansible_host: <host1-ip>
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.9-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: <wheel-file>
 ```
 
 #### Example: Patch Upgrade Inventory
@@ -47,8 +47,8 @@ all:
         <host1:
           ansible_host: <host1-ip>
       vars:
-        gateway_release: 2023.1
-        gateway_whl_file: automation_gateway-3.227.0+2023.1.52-py3-none-any.whl
+        gateway_release: 4.3
+        gateway_whl_file: <wheel-file>
 ```
 
 ### Run Patch IAG Playbook

roles/gateway/tasks/create-gateway-archive.yml

@@ -27,13 +27,6 @@
     state: present
     virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_install"
 
-- name: Install Ansible
-  ansible.builtin.pip:
-    name: "{{ gateway_ansible_version }}"
-    state: present
-    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_install"
-  when: gateway_enable_ansible | bool
-
 - name: Copy IAG wheel file to target node
   ansible.builtin.copy:
     src: "{{ gateway_whl_file }}"

roles/gateway/tasks/download-build-rpms.yml

@@ -1,23 +1,6 @@
 # Copyright (c) 2024, Itential, Inc
 # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt)
 ---
-- name: Download Gateway build RPMs
-  ansible.builtin.import_role:
-    name: offline
-    tasks_from: download-rpms
-  vars:
-    offline_download_method: yum_module
-    offline_download_packages: "{{ gateway_build_packages }}"
-    offline_download_dir: "{{ gateway_offline_target_node_rpms_dir }}/build"
-
-- name: Copy Gateway build RPMs to control node
-  ansible.builtin.import_role:
-    name: offline
-    tasks_from: fetch-packages
-  vars:
-    offline_src_dir: "{{ gateway_offline_target_node_rpms_dir }}/build"
-    offline_dest_dir: "{{ gateway_offline_control_node_rpms_dir }}/build"
-
 - name: Install Gateway build packages
   ansible.builtin.dnf:
     name: "{{ item }}"

roles/gateway/tasks/download-dependency-wheels.yml

@@ -38,37 +38,31 @@
 - name: Setup Python virtual environment
   ansible.builtin.command:
     chdir: "{{ gateway_pkgs_temp_dir.path }}"
-    cmd: "{{ gateway_python_executable }} -m venv offline_install"
+    cmd: "{{ gateway_python_executable }} -m venv offline_download"
   changed_when: true
 
-- name: Install base Python Dependencies
+- name: Install base Python dependencies
   ansible.builtin.pip:
     name: "{{ gateway_python_base_dependencies }}"
     state: present
-    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_install"
+    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_download"
     extra_args: --log /var/log/pip.log
   when:
     - gateway_python_base_dependencies is defined
+    - gateway_python_base_dependencies is iterable
     - gateway_python_base_dependencies | length > 0
 
-- name: Install IAG Python Dependencies
+- name: Install IAG Python dependencies
   ansible.builtin.pip:
     name: "{{ gateway_python_app_dependencies }}"
     state: present
-    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_install"
+    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_download"
     extra_args: --log /var/log/pip.log
   when:
     - gateway_python_app_dependencies is defined
+    - gateway_python_app_dependencies is iterable
     - gateway_python_app_dependencies | length > 0
 
-- name: Install Ansible
-  ansible.builtin.pip:
-    name: "{{ gateway_ansible_version }}"
-    state: present
-    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_install"
-    extra_args: --log /var/log/pip.log
-  when: gateway_enable_ansible | bool
-
 - name: Download Ansible collections
   ansible.builtin.include_tasks:
     file: download-ansible-collections.yml
@@ -77,33 +71,39 @@
     - gateway_ansible_collections is iterable
     - gateway_ansible_collections | length > 0
 
-- name: Install IAG
-  ansible.builtin.pip:
-    name: "{{ gateway_wheel_download.files[0].path }}"
-    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_install"
-    extra_args: --log /var/log/pip.log
-
-- name: Uninstall IAG
-  ansible.builtin.pip:
-    name: "{{ gateway_wheel_download.files[0].path }}"
-    state: absent
-    virtualenv: "{{ gateway_pkgs_temp_dir.path }}/offline_install"
-    extra_args: --log /var/log/pip.log
-
-- name: Create requirements file
-  ansible.builtin.shell:
+- name: Build wheels from source distributions
+  ansible.builtin.command:
+    cmd: >
+      offline_download/bin/pip3 wheel
+      {{ item }}
+      -w {{ gateway_offline_target_node_wheels_dir }}/app
+      --no-deps
+  args:
     chdir: "{{ gateway_pkgs_temp_dir.path }}"
-    cmd: offline_install/bin/pip3 freeze > requirements.txt
+  loop: "{{ gateway_python_wheel_build_dependencies }}"
   changed_when: true
 
-- name: Download wheel files
+- name: Download automation-gateway and remaining dependencies
   ansible.builtin.command:
-    cmd: "offline_install/bin/pip3 download -r requirements.txt
-         -d {{ gateway_offline_target_node_wheels_dir }}/app"
+    cmd: >
+      offline_download/bin/pip3 download {{ gateway_wheel_download.files[0].path }}
+      -d {{ gateway_offline_target_node_wheels_dir }}/app
   args:
     chdir: "{{ gateway_pkgs_temp_dir.path }}"
   changed_when: true
 
+- name: Find source distributions after building wheels
+  ansible.builtin.find:
+    paths: "{{ gateway_offline_target_node_wheels_dir }}/app"
+    patterns: "*.tar.gz"
+  register: gateway_source_dists
+
+- name: Delete source distributions
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ gateway_source_dists.files }}"
+
 - name: Copy IAG Python dependencies to control node
   ansible.builtin.import_role:
     name: offline

roles/gateway/tasks/main.yml

@@ -124,39 +124,6 @@
             - gateway_build_packages is iterable
             - gateway_build_packages | length > 0
 
-        - name: Install Gateway build packages (offline)
-          when: offline_install_enabled | bool
-          block:
-            - name: Install Gateway build packages (offline)
-              ansible.builtin.import_role:
-                name: offline
-                tasks_from: install-rpms
-              vars:
-                offline_rpms_path: "{{ gateway_offline_control_node_rpms_dir }}/build"
-
-            # Build a list of installed build packages. The rpm_list variable is registered in the
-            # offline install-rpms.
-            - name: Set Gateway build package list
-              ansible.builtin.set_fact:
-                gateway_installed_build_package_list:
-                  "{{ gateway_installed_build_package_list | default([]) +
-                  [item | basename | split('.el') | first] }}"
-              with_items: "{{ rpm_list.files | map(attribute='path') | list | sort }}"
-              when:
-                - rpm_list is defined
-                - rpm_list is iterable
-                - rpm_list | length > 0
-
-            # Build a list of build packages that need to be removed later. We don't want to remove
-            # all of the installed build packages, only the ones that are in the gateway_build_packages
-            # and gateway_installed_build_package_list.
-            - name: Set Gateway build package remove list
-              ansible.builtin.set_fact:
-                gateway_build_package_remove_list:
-                  "{{ gateway_build_package_remove_list | default([]) + [item] }}"
-              with_items: "{{ gateway_build_packages }}"
-              when: gateway_installed_build_package_list | select('search', item | regex_escape) | list | length > 0
-
     - name: Install Python dependencies
       tags: install_python_dependencies
       block:
@@ -165,18 +132,13 @@
             file: install-python-dependencies.yml
 
     # Need to install ansible within the virtual environment
-    - name: Install Ansible
-      tags: install_ansible
+    - name: Configure Ansible
+      tags: configure_ansible
       when: gateway_enable_ansible | bool
       block:
-        - name: Install Ansible and Ansible collections
+        - name: Install Ansible collections
           when: not offline_install_enabled | bool
           block:
-            - name: Install Ansible into the python virtual environment
-              ansible.builtin.pip:
-                name: "{{ gateway_ansible_version }}"
-                virtualenv: "{{ gateway_install_dir }}/venv"
-
             - name: Install collections
               ansible.builtin.command:
                 argv:
@@ -279,15 +241,6 @@
         virtualenv: "{{ gateway_install_dir }}/venv"
       when: not offline_install_enabled | bool
 
-    - name: Install IAG (offline)
-      ansible.builtin.include_role:
-        name: offline
-        tasks_from: install-wheels
-      vars:
-        offline_wheels_dir: "{{ gateway_offline_control_node_wheels_dir }}/gateway"
-        offline_python_venv: "{{ gateway_python_venv }}"
-      when: offline_install_enabled | bool
-
     - name: Set ownership/permissions and create properties.yml
       when: not gateway_installed.stat.exists
       block:
@@ -423,25 +376,6 @@
             - gateway_build_packages_result.results is iterable
             - gateway_build_packages_result.results | length > 0
 
-        - name: Uninstall Gateway build packages (offline)
-          when: offline_install_enabled | bool
-          block:
-            - name: Get list of yum repos (to disable temporarily) # noqa command-instead-of-module
-              ansible.builtin.command: yum -q repolist
-              register: gateway_repolist_result
-              changed_when: false
-
-            - name: Uninstall Gateway build packages (offline)
-              ansible.builtin.dnf:
-                name: "{{ gateway_build_package_remove_list }}"
-                state: absent
-                enablerepo: []
-                disablerepo: "{{ gateway_repolist_result.stdout_lines[1:] | map('split', ' ')
-                              | map('first') | list }}"
-              when:
-                - gateway_build_package_remove_list is defined
-                - gateway_build_package_remove_list | length > 0
-
     - name: Assert that Gateway is running
       ansible.builtin.systemd:
         name: automation-gateway

roles/gateway/tasks/update-release-file.yml

@@ -38,13 +38,25 @@
   changed_when: true
 
 - name: Update release file with Ansible version
-  ansible.builtin.lineinfile:
-    path: "{{ common_itential_release_file }}"
-    line: "ANSIBLE={{ gateway_ansible_version }}"
-    mode: "0666"
-    create: true
   when: gateway_enable_ansible | bool
-  changed_when: true
+  block:
+    - name: Get Ansible version from automation-gateway venv
+      ansible.builtin.command:
+        cmd: "{{ gateway_python_venv }}/bin/ansible --version"
+      register: gateway_ansible_venv_version
+      changed_when: false
+
+    - name: Extract version
+      ansible.builtin.set_fact:
+        gateway_ansible_version: "{{ gateway_ansible_venv_version.stdout_lines[0] | regex_search('\\d+\\.\\d+\\.\\d+') }}"
+
+    - name: Update release file with Ansible version
+      ansible.builtin.lineinfile:
+        path: "{{ common_itential_release_file }}"
+        line: "ANSIBLE={{ gateway_ansible_version }}"
+        mode: "0666"
+        create: true
+      changed_when: true
 
 - name: Update release file with Nornir version
   ansible.builtin.lineinfile: