fix: release 1.1.2

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 ### Changed
+- add fallback for MONGODB and REDIS connection strings, fix logging
+- create MONGODB connection string from environment variables instead of full MONGO_URI variable
 - create REDIS connection string from environment variables instead of full REDIS_URL variable
 
 ## 1.1.0

backend/Dockerfile

@@ -6,14 +6,18 @@ COPY SC4SNMP_UI_backend ./SC4SNMP_UI_backend
 RUN pip install -r ./requirements.txt
 ENV FLASK_DEBUG production
 
-
+# Copy scripts
 COPY ./flask_start.sh /flask_start.sh
-RUN chmod +x /flask_start.sh
-
 COPY ./celery_start.sh /celery_start.sh
-RUN chmod +x /celery_start.sh
+COPY construct-connection-strings.sh /app/construct-connection-strings.sh
 
+# Set permissions BEFORE USER switch
+RUN chmod +x /flask_start.sh /celery_start.sh /app/construct-connection-strings.sh
+
+# Switch to non-root
 USER 10000:10000
 
 EXPOSE 5000
-CMD ["gunicorn", "-b", ":5000", "app:flask_app", "--log-level", "INFO"]
+
+# Use flask_start.sh which sources construct-connection-strings.sh
+CMD ["/flask_start.sh"]

backend/SC4SNMP_UI_backend/__init__.py

@@ -1,5 +1,9 @@
+import sys
+import time
+
 from flask import Flask
 from pymongo import MongoClient
+from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError
 import os
 import logging
 from celery import Celery
@@ -8,28 +12,68 @@
 
 load_dotenv()
 
-__version__ = "1.1.1"
+__version__ = "1.1.2-beta.3"
 
 MONGO_URI = os.getenv("MONGO_URI")
+log = logging.getLogger('gunicorn.error')
+log.setLevel(logging.INFO)
+
+def wait_for_mongodb_replicaset(logger, mongo_uri, max_retries=120, retry_interval=5):
+    """
+    Wait for MongoDB to be ready before starting the application.
+    For replica sets, waits for PRIMARY to be elected.
+    """
+    mongo_mode = os.getenv("MONGODB_MODE", "standalone").lower()
+    if mongo_mode == "standalone":
+        logger.info("MongoDB is in standalone mode, skipping ReplicaSet wait")
+        return
+
+    if not mongo_uri:
+        logger.warning("MONGO_URI not set, exiting application")
+        sys.exit(1)
+
+    logger.info(f"Waiting for MongoDB ReplicaSet to be ready and elect the primary...")
+
+    for attempt in range(1, max_retries + 1):
+        if attempt != 1:
+            time.sleep(retry_interval)
+        try:
+            # Try to connect
+            client = MongoClient(
+                mongo_uri, serverSelectionTimeoutMS=5000, connectTimeoutMS=5000
+            )
+
+            # Execute a simple operation to verify PRIMARY exists
+            client.admin.command("ping")
+
+            # For replica sets, verify PRIMARY exists
+            if "replicaSet=" in mongo_uri:
+                if client.primary is None:
+                    continue
+                logger.info(f"PRIMARY found: {client.primary}")
+
+            client.close()
+            logger.info("MongoDB is ready")
+            return
+
+        except (ServerSelectionTimeoutError, ConnectionFailure, Exception) as e:
+            if attempt >= max_retries:
+                logger.info(f"MongoDB not ready after {max_retries * retry_interval}s")
+                logger.info(f"   Error: {e}")
+                sys.exit(1)
+
+        logger.info(f"  Still waiting... ({attempt}/{max_retries})")
+
+wait_for_mongodb_replicaset(log, MONGO_URI)
 mongo_client = MongoClient(MONGO_URI)
 
 VALUES_DIRECTORY = os.getenv("VALUES_DIRECTORY", "")
 KEEP_TEMP_FILES = os.getenv("KEEP_TEMP_FILES", "false")
 
-REDIS_HOST = os.getenv("REDIS_HOST", "snmp-redis")
-REDIS_PORT = os.getenv("REDIS_PORT", "6379")
-REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
-REDIS_DB = os.getenv("REDIS_DB", "1")
-CELERY_DB = os.getenv("CELERY_DB", "0")
-
-if REDIS_PASSWORD:
-    redis_base = f"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}"
-else:
-    redis_base = f"redis://{REDIS_HOST}:{REDIS_PORT}"
-
-# fallback
-REDBEAT_URL = os.getenv("REDIS_URL", f"{redis_base}/{REDIS_DB}")
-CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", f"{redis_base}/{CELERY_DB}")
+REDBEAT_URL = os.getenv("REDIS_URL", "redis://snmp-redis-headless:6379")
+CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "sentinel://snmp-redis-sentinel:26379")
+REDIS_SENTINEL_SERVICE = os.getenv("REDIS_SENTINEL_SERVICE", "snmp-redis-sentinel")
+REDIS_MODE = os.getenv("REDIS_MODE", "standalone")
 
 
 class NoValuesDirectoryException(Exception):
@@ -41,17 +85,38 @@ def create_app():
 
     app = Flask(__name__)
 
+    if REDIS_MODE == "replication":
+        broker_transport_options = {
+            "priority_steps": list(range(10)),
+            "sep": ":",
+            "queue_order_strategy": "priority",
+            "service_name": "mymaster",
+            "master_name": "mymaster",
+            "socket_timeout": 5,
+            "retry_policy": {
+                "max_retries": 100,
+                "interval_start": 0,
+                "interval_step": 2,
+                "interval_max": 5,
+            },
+            "db": 1,
+            "sentinels": [(REDIS_SENTINEL_SERVICE, 26379)],
+            "password": os.getenv("REDIS_PASSWORD", None),
+        }
+    else:
+        broker_transport_options = {
+            "priority_steps": list(range(10)),
+            "sep": ":",
+            "queue_order_strategy": "priority"
+        }
+
     app.config.from_mapping(
         CELERY=dict(
             task_default_queue="apply_changes",
             broker_url=CELERY_BROKER_URL,
             beat_scheduler="redbeat.RedBeatScheduler",
             redbeat_redis_url = REDBEAT_URL,
-            broker_transport_options={
-                "priority_steps": list(range(10)),
-                "sep": ":",
-                "queue_order_strategy": "priority",
-            },
+            broker_transport_options=broker_transport_options,
             task_ignore_result=True,
             redbeat_lock_key=None,
         ),

backend/celery_start.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+set -e
+
+# Source Redis URL construction
+. /app/construct-connection-strings.sh
+
 set -o errexit
 set -o nounset
 
 cd /app
-celery -A app worker -Q apply_changes --loglevel INFO
+celery -A app worker -Q apply_changes --loglevel INFO

backend/construct-connection-strings.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env sh
+# Constructs REDIS_URL and CELERY_BROKER_URL from components if not already set
+
+# Detect mode
+REDIS_MODE="${REDIS_MODE:-standalone}"
+
+# Only construct if URLs not already set
+if [ -z "$REDIS_URL" ] || [ -z "$CELERY_BROKER_URL" ]; then
+
+  if [ "$REDIS_MODE" = "replication" ]; then
+    # Sentinel HA mode
+    echo "Redis mode: Sentinel HA"
+
+    REDIS_SENTINEL_SERVICE="${REDIS_SENTINEL_SERVICE:-snmp-redis-sentinel}"
+    REDIS_HEADLESS_SERVICE="${REDIS_HEADLESS_SERVICE:-snmp-redis-headless}"
+    REDIS_SENTINEL_PORT="${REDIS_SENTINEL_PORT:-26379}"
+    REDIS_PORT="${REDIS_PORT:-6379}"
+    REDIS_MASTER_NAME="${REDIS_MASTER_NAME:-mymaster}"
+    REDIS_DB="${REDIS_DB:-1}"
+    CELERY_DB="${CELERY_DB:-0}"
+
+    if [ -n "$REDIS_PASSWORD" ]; then
+      SENTINEL_SCHEME="sentinel://:${REDIS_PASSWORD}@${REDIS_SENTINEL_SERVICE}:${REDIS_SENTINEL_PORT}"
+      REDIS_HA_CHECK="redis://:${REDIS_PASSWORD}@${REDIS_HEADLESS_SERVICE}:${REDIS_PORT}"
+    else
+      SENTINEL_SCHEME="sentinel://${REDIS_SENTINEL_SERVICE}:${REDIS_SENTINEL_PORT}"
+      REDIS_HA_CHECK="redis://${REDIS_HEADLESS_SERVICE}:${REDIS_PORT}"
+    fi
+
+    REDBEAT_SCHEME="redis-sentinel://${REDIS_SENTINEL_SERVICE}:${REDIS_SENTINEL_PORT}"
+
+    # Celery broker uses sentinel://
+    : "${CELERY_BROKER_URL:=${SENTINEL_SCHEME}/${CELERY_DB}#master_name=${REDIS_MASTER_NAME}}"
+
+    # RedBeat uses redis-sentinel:// with master_name query
+    : "${REDIS_URL:=${REDBEAT_SCHEME}/${REDIS_DB}#master_name=${REDIS_MASTER_NAME}}"
+
+  else
+    # Standalone mode
+    echo "Redis mode: Standalone"
+    REDIS_HOST="${REDIS_HOST:-snmp-redis}"
+    REDIS_PORT="${REDIS_PORT:-6379}"
+
+    if [ -n "$REDIS_PASSWORD" ]; then
+      BASE="redis://:$REDIS_PASSWORD@$REDIS_HOST:$REDIS_PORT"
+    else
+      BASE="redis://$REDIS_HOST:$REDIS_PORT"
+    fi
+
+    : "${REDIS_URL:=$BASE/${REDIS_DB:-1}}"
+    : "${CELERY_BROKER_URL:=$BASE/${CELERY_DB:-0}}"
+  fi
+  export REDIS_URL
+  export CELERY_BROKER_URL
+  export REDIS_MODE
+fi
+
+# Only construct if MONGO_URI not already set
+if [ -z "$MONGO_URI" ]; then
+  # Build MongoDB URI from environment variables
+  if [ -n "$MONGODB_PASSWORD" ]; then
+    # With authentication
+    if [ -n "$MONGODB_REPLICA_SET" ]; then
+      # Replica set
+      export MONGO_URI="mongodb://${MONGODB_USERNAME}:${MONGODB_PASSWORD}@${MONGODB_HOST}/${MONGODB_DATABASE}?replicaSet=${MONGODB_REPLICA_SET}&authSource=${MONGODB_AUTH_SOURCE:-admin}&readPreference=primary"
+    else
+      # Standalone
+      export MONGO_URI="mongodb://${MONGODB_USERNAME}:${MONGODB_PASSWORD}@${MONGODB_HOST}:${MONGODB_PORT}/${MONGODB_DATABASE}?authSource=${MONGODB_AUTH_SOURCE:-admin}"
+    fi
+  else
+    # Without authentication
+    if [ -n "$MONGODB_REPLICA_SET" ]; then
+      export MONGO_URI="mongodb://${MONGODB_HOST}/${MONGODB_DATABASE}?replicaSet=${MONGODB_REPLICA_SET}&authSource=${MONGODB_AUTH_SOURCE:-admin}&retryWrites=false&readPreference=primary"
+    else
+      export MONGO_URI="mongodb://${MONGODB_HOST}:${MONGODB_PORT}/${MONGODB_DATABASE}?authSource=${MONGODB_AUTH_SOURCE:-admin}"
+    fi
+  fi
+
+  export MONGO_URI
+
+fi

backend/flask_start.sh

@@ -1,4 +1,11 @@
+#!/bin/bash
+set -e
+
+# Source Redis URL construction
+. /app/construct-connection-strings.sh
+
 set -o errexit
 set -o nounset
-cd /app
-gunicorn -b :5000 app:flask_app --log-level INFO
+
+# Start Gunicorn
+exec gunicorn -b :5000 app:flask_app --log-level INFO

backend/requirements.txt

@@ -13,5 +13,5 @@ kubernetes~=26.1.0
 python-dotenv~=0.21.0
 PyYAML~=6.0
 celery==5.5.3
-redis==4.5.5
+redis~=4.5, !=4.5.5
 ruamel.yaml===0.17.32

frontend/lerna.json

@@ -1,6 +1,6 @@
 {
   "lerna": "^6.6.2",
-  "version": "1.1.1",
+  "version": "1.1.2-beta.3",
   "command": {
     "publish": {
       "ignoreChanges": ["*.md"]

frontend/packages/manager/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@splunk/manager",
-  "version": "1.1.1",
+  "version": "1.1.2-beta.3",
   "license": "UNLICENSED",
   "scripts": {
     "build": "NODE_ENV=production webpack --bail --config demo/webpack.standalone.config.js",