chore: ADD escu tests for spl2 transformed logs

Author: spanchal-crest

.github/workflows/reusable-build-test-release.yml

@@ -123,6 +123,7 @@ jobs:
     outputs:
       execute-knowledge-labeled: ${{ steps.configure-tests-on-labels.outputs.execute_knowledge_labeled }}
       execute-escu-labeled: ${{ steps.configure-tests-on-labels.outputs.execute_escu_labeled }}
+      execute-spl2-escu-labeled: ${{ steps.configure-tests-on-labels.outputs.execute_spl2_escu_labeled }}
       execute-ui-labeled: ${{ steps.configure-tests-on-labels.outputs.execute_ui_labeled }}
       execute-modinput-labeled: ${{ steps.configure-tests-on-labels.outputs.execute_modinput_functional_labeled }}
       execute-ucc-modinput-labeled: ${{ steps.configure-tests-on-labels.outputs.execute_ucc_modinput_functional_labeled }}
@@ -157,7 +158,7 @@ jobs:
         run: |
           set +e
           declare -A EXECUTE_LABELED
-          TESTSET=("execute_knowledge" "execute_escu" "execute_ui" "execute_modinput_functional" "execute_ucc_modinput_functional" "execute_scripted_inputs" "execute_requirement_test" "execute_upgrade")
+          TESTSET=("execute_knowledge" "execute_escu" "execute_spl2_escu" "execute_ui" "execute_modinput_functional" "execute_ucc_modinput_functional" "execute_scripted_inputs" "execute_requirement_test" "execute_upgrade")
           for test_type in "${TESTSET[@]}"; do
             EXECUTE_LABELED["$test_type"]="false"
           done
@@ -426,6 +427,155 @@ jobs:
           import yaml
           import os
           import configparser 
+          import re
+
+          # Parse app.conf get the appid of the TA.
+          config = configparser.ConfigParser(strict=False)
+          config.read("package/default/app.conf")
+          APP_ID = config.get("id", "name")
+
+          # Read the file and remove trailing backslashes
+          with open("package/default/props.conf", "r") as f:
+              content = f.read()
+
+          # Remove trailing backslashes followed by a newline
+          updated_content = re.sub(r"\\\n", "", content)
+
+          # Write the cleaned content to a new file
+          with open("package/default/props.conf", "w") as f:
+              f.write(updated_content)
+
+          # Parse props.conf and collect all the sourcetypes in a list.
+          config = configparser.ConfigParser(strict=False)
+          config.read("package/default/props.conf")
+          sourcetypes = config.sections()
+
+          # Load the YAML content
+          with open("security_content/contentctl.yml", "r") as file:
+              data = yaml.safe_load(file)
+
+              for app in data["apps"]:
+                  if app['appid'] == APP_ID:
+                      app['hardcoded_path'] = ADDON_URL
+
+          # Write the modified data to the contentctl.yml file
+          with open("security_content/contentctl.yml", "w") as file:
+              yaml.dump(data,file,sort_keys=False)
+
+          # Filter out the detections based on the collected sourcetypes
+          base_dir = "security_content/detections"
+          detection_files = ""
+
+          for root, dirs, files in os.walk(base_dir):
+              for file in files:
+                  file_path = os.path.join(root, file)
+
+                  try:
+                      with open(file_path, "r") as file:
+                          file_content = yaml.safe_load(file)
+                          if file_content["tests"][0]["attack_data"][0]["sourcetype"] in sourcetypes or file_content["tests"][0]["attack_data"][0]["source"] in sourcetypes:
+                              detection_files += file_path.replace("security_content/", "") + " "
+                              
+
+                  except Exception as e:
+                      continue
+
+          # Save detection_files as an output variable
+          with open(os.getenv('GITHUB_OUTPUT'), 'w') as output_file:
+              output_file.write(f"DETECTION_FILES={detection_files}")
+
+          print(f"Filtered Detection files = {detection_files}")
+
+      - name: Run ESCU Tests
+        run: |
+
+          cd security_content
+          echo "Content of contentctl.yml file"
+          cat contentctl.yml
+
+          echo "contentctl test --container-settings.num-containers 4 --post-test-behavior never_pause --verbose --container-settings.no-leave-running mode:selected --mode.files ${{ steps.filter-detection-files.outputs.DETECTION_FILES }}"
+
+          contentctl test --container-settings.num-containers 4 --post-test-behavior never_pause --verbose --container-settings.no-leave-running mode:selected --mode.files ${{ steps.filter-detection-files.outputs.DETECTION_FILES }}
+      
+      - uses: actions/upload-artifact@v4
+        with:
+          name: escu_test_summary_results
+          path: |
+            security_content/test_results/summary.yml
+            security_content/dist/DA-ESS-ContentUpdate-latest.tar.gz
+        continue-on-error: true
+
+  run-spl2-escu-tests:
+    if: ${{ !cancelled() && needs.setup-workflow.outputs.execute-spl2-escu-labeled == 'true' }}
+    needs:
+      - build
+      - setup-workflow
+      - setup
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.11"
+    permissions:
+      actions: read
+      deployments: read
+      contents: read
+      packages: read
+      statuses: read
+      checks: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Python Dependencies and ContentCTL
+        run: |
+          python -m pip install --upgrade pip
+          pip install contentctl==5.0.0
+          git clone https://github.com/splunk/security_content.git
+
+
+      - name: Download TA Build Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: package-splunkbase
+          path: ta_build 
+
+      - name: Get the build path
+        run: |
+          TA_BUILD=$(ls ta_build)
+          TA_BUILD_PATH="${{ github.workspace }}/ta_build/$TA_BUILD"
+          echo "TA_BUILD_PATH=$TA_BUILD_PATH" >> $GITHUB_ENV
+
+          pwd
+
+          echo ',,978075574018620,TRAFFIC,drop,10.0,,141.151.140.150,198.0.181.90,0.0.0.0,0.0.0.0,Advanced-human-resource-product,davis/duncanashley,,nielsenrebecca,vsys1,Streamlined-asymmetric-methodology,User-friendly-next-generation-structure,,,fL-logger,,1452,1,56710,389,0,0,5002,udp,allow,6134,3098,3036,7623,2022/02/23 20:30:51,5807,any,,3053,-216944275885572,ZW,ID,,6153,1470,Unknown,0,0,0,0,,PA-uw,Unknown,,,0,,0,,hrs-oL-wDV' > detection.log
+
+          pwd
+          ls
+
+      - name: Run Python Script
+        id: filter-detection-files
+        shell: python
+        run: |
+          import yaml
+          import os
+          import configparser 
+          import re
+
+          # Read the file and remove trailing backslashes
+          with open("package/default/props.conf", "r") as f:
+              content = f.read()
+
+          # Remove trailing backslashes followed by a newline
+          updated_content = re.sub(r"\\\n", "", content)
+
+          # Write the cleaned content to a new file
+          with open("package/default/props.conf", "w") as f:
+              f.write(updated_content)
 
           # Parse props.conf and collect all the sourcetypes in a list.
           config = configparser.ConfigParser(strict=False)
@@ -436,12 +586,24 @@ jobs:
           with open("security_content/contentctl.yml", "r") as file:
               data = yaml.safe_load(file)
 
+              data["container_settings"]["full_image_path"] = data["container_settings"]["full_image_path"].replace("9.3", "9.4")
+
               data["apps"] = [{'uid': 1621, 'title': "Splunk Common Information Model (CIM)", 'version': "6.0.1", 'appid': "Splunk_SA_CIM",  'hardcoded_path': "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/splunk-common-information-model-cim_601.tgz"}, {'title': ${{ needs.setup.outputs.addon-name }}, 'version': "default", 'appid': ${{ needs.setup.outputs.addon-name }}, 'hardcoded_path': "${{ env.TA_BUILD_PATH }}"}]
 
           # Write the modified data to the contentctl.yml file
           with open("security_content/contentctl.yml", "w") as file:
               yaml.dump(data,file,sort_keys=False)
 
+          # Load the YAML content
+          with open("security_content/detections/network/detect_outbound_ldap_traffic.yml", "r") as file:
+              data = yaml.safe_load(file)
+
+              data["tests"][0]["attack_data"][0]["data"] = "${{ github.workspace }}/detection.log"
+
+          # Write the modified data to the contentctl.yml file
+          with open("security_content/detections/network/detect_outbound_ldap_traffic.yml", "w") as file:
+              yaml.dump(data,file,sort_keys=False)
+
           # Filter out the detections based on the collected sourcetypes
           base_dir = "security_content/detections"
           detection_files = ""
@@ -470,12 +632,25 @@ jobs:
         run: |
 
           cd security_content
+
           echo "Content of contentctl.yml file"
           cat contentctl.yml
 
-          echo "contentctl test --post-test-behavior never_pause --verbose --container-settings.no-leave-running mode:selected --mode.files ${{ steps.filter-detection-files.outputs.DETECTION_FILES }}"
+          echo "Content of etect_outbound_ldap_traffic.yml file"
+          cat detections/network/detect_outbound_ldap_traffic.yml
+
+          echo "contentctl test --post-test-behavior never_pause --verbose --container-settings.no-leave-running mode:selected --mode.files detections/network/detect_outbound_ldap_traffic.yml"
 
-          contentctl test --post-test-behavior never_pause --verbose --container-settings.no-leave-running mode:selected --mode.files ${{ steps.filter-detection-files.outputs.DETECTION_FILES }}
+          contentctl test --post-test-behavior never_pause --verbose --container-settings.no-leave-running mode:selected --mode.files detections/network/detect_outbound_ldap_traffic.yml
+      
+      - uses: actions/upload-artifact@v4
+        with:
+          name: spl2_escu_test_summary_results
+          path: |
+            test_results/summary.yml
+            dist/DA-ESS-ContentUpdate-latest.tar.gz
+        continue-on-error: true
+      
 
   run-unit-tests:
     name: test-unit-python3-${{ matrix.python-version }}