introduce a SystemEnv class to unify environment variable use and enable smooth transition

collectoss/api/gunicorn_conf.py

@@ -7,6 +7,7 @@
 
 from collectoss.application.db.lib import get_value
 from collectoss.application.db import dispose_database_engine
+from collectoss.application.environment import SystemEnv
 
 logger = logging.getLogger(__name__)
 
@@ -20,8 +21,8 @@
 workers = multiprocessing.cpu_count() * 2 + 1
 umask = 0o007
 reload = True
-
-is_dev = os.getenv("AUGUR_DEV", 'False').lower() in ('true', '1', 't', 'y', 'yes')
+# this satisfies the type checker
+is_dev = SystemEnv.get_bool("AUGUR_DEV", False)
 
 if is_dev:
 
@@ -40,7 +41,8 @@
 # set the log location for gunicorn    
 logs_directory = get_value('Logging', 'logs_directory')
 
-is_docker = os.getenv("AUGUR_DOCKER_DEPLOY", 'False').lower() in ('true', '1', 't', 'y', 'yes')
+# this syntax satisfies the type checker
+is_docker = SystemEnv.get_bool("AUGUR_DOCKER_DEPLOY", False)
 accesslog = f"{logs_directory}/gunicorn.log"
 errorlog = f"{logs_directory}/gunicorn.log"
 

collectoss/api/routes/auggie.py

@@ -14,6 +14,8 @@
 import requests
 import slack
 
+from collectoss.application.environment import SystemEnv
+
 from ..server import app
 
 
@@ -252,7 +254,7 @@ def get_auggie_user():
     # return Response(response=response, status=200, mimetype="application/json")
     ## From Method
     profile_name = 'collectoss'
-    if os.environ.get('AUGUR_IS_PROD'):
+    if SystemEnv.get('AUGUR_IS_PROD'):
         profile_name = 'default'
     client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb')
     response = client.get_item(
@@ -278,7 +280,7 @@ def update_auggie_user_tracking():
     # return Response(response=response, status=200, mimetype="application/json")
     ## From Method
     profile_name = 'collectoss'
-    if os.environ.get('AUGUR_IS_PROD'):
+    if SystemEnv.get('AUGUR_IS_PROD'):
         profile_name = 'default'
     client = boto3.Session(region_name='us-east-1', profile_name=profile_name).client('dynamodb')
     response = client.update_item(
@@ -326,7 +328,7 @@ def slack_login():
     print("slack_login")
 
     r = requests.get(
-        url=f'https://slack.com/api/oauth.v2.access?code={body["code"]}&client_id={os.environ["AUGGIE_CLIENT_ID"]}&client_secret={os.environ["AUGGIE_CLIENT_SECRET"]}&redirect_uri=http%3A%2F%2Flocalhost%3A8080')
+        url=f'https://slack.com/api/oauth.v2.access?code={body["code"]}&client_id={SystemEnv.get("AUGGIE_CLIENT_ID")}&client_secret={SystemEnv.get("AUGGIE_CLIENT_SECRET")}&redirect_uri=http%3A%2F%2Flocalhost%3A8080')
     data = r.json()
 
     if (data["ok"]):
@@ -340,7 +342,7 @@ def slack_login():
         email = user_response["user"]["email"]
 
         profile_name = 'collectoss'
-        if os.environ.get('AUGUR_IS_PROD'):
+        if SystemEnv.get('AUGUR_IS_PROD'):
             profile_name = 'default'
         print("Making Boto3 Session")
         client = boto3.Session(region_name='us-east-1',

collectoss/api/view/init.py

@@ -1,13 +1,11 @@
 import os
 from pathlib import Path
-from .server import Environment
 from collectoss.application.logs import SystemLogger
 import secrets, yaml
-
-env = Environment()
+from collectoss.application.environment import SystemEnv
 
 # load configuration files and initialize globals
-configFile = Path(env.setdefault("CONFIG_LOCATION", "config.yml"))
+configFile = Path(SystemEnv.get("CONFIG_LOCATION") or "config.yml")
 
 settings = {}
 

collectoss/api/view/server/__init__.py

@@ -1,2 +1 @@
-from .LoginException import LoginException
-from .Environment import Environment
+from .LoginException import LoginException

collectoss/application/cli/__init__.py

@@ -10,7 +10,9 @@
 
 from collectoss.application.db.engine import DatabaseEngine
 from collectoss.application.db import get_engine, dispose_database_engine
-from sqlalchemy.exc import OperationalError 
+from sqlalchemy.exc import OperationalError
+from collectoss.application.environment import SystemEnv
+
 
 
 def check_connectivity(urls=["http://chaoss.community", "http://github.com", "http://gitlab.com"], timeout=10.0):
@@ -65,11 +67,11 @@ def new_func(ctx, *args, **kwargs):
             return ctx.invoke(function_db_connection, *args, **kwargs)
         except OperationalError as e:
 
-            db_environment_var = os.getenv("AUGUR_DB")
+            db_environment_var = SystemEnv.get("AUGUR_DB")
 
             # determine the location to print in error string
             if db_environment_var:
-                location = f"the AUGUR_DB environment variable\nAUGUR_DB={os.getenv('AUGUR_DB')}"
+                location = f"the AUGUR_DB environment variable\nAUGUR_DB={SystemEnv.get('AUGUR_DB')}"
             else:
                 with open("db.config.json", 'r') as f:
                     db_config = json.load(f)

collectoss/application/cli/api.py

@@ -17,6 +17,8 @@
 from collectoss.application.cli import test_connection, test_db_connection, with_database, DatabaseContext
 from collectoss.application.cli._cli_util import _broadcast_signal_to_processes, raise_open_file_limit, clear_redis_caches, clear_rabbitmq_messages
 from collectoss.application.db.lib import get_value
+from collectoss.application.environment import SystemEnv
+
 
 logger = SystemLogger("collectoss", reset_logfiles=False).get_logger()
 
@@ -36,7 +38,7 @@ def start(ctx, development, port):
     """Start CollectOSS's backend server."""
 
     try:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1":
             raise_open_file_limit(100000)
     except Exception as e: 
         logger.error(
@@ -46,7 +48,7 @@ def start(ctx, development, port):
         raise e
 
     if development:
-        os.environ["AUGUR_DEV"] = "1"
+        SystemEnv.set("AUGUR_DEV", "1")
         logger.info("Starting in development mode")
 
     try:
@@ -142,7 +144,7 @@ def get_api_processes():
 def is_api_process(process):
 
     command = ''.join(process.info['cmdline'][:]).lower()
-    if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
+    if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
 
         if process.pid != os.getpid():
 

collectoss/application/cli/backend.py

@@ -15,6 +15,7 @@
 import requests
 from redis.exceptions import ConnectionError as RedisConnectionError
 
+from collectoss.application.environment import SystemEnv
 from collectoss.tasks.start_tasks import collection_monitor, create_collection_status_records
 from collectoss.tasks.git.facade_tasks import clone_repos
 from collectoss.tasks.github.contributors import process_contributors
@@ -31,7 +32,7 @@
 
 from keyman.KeyClient import KeyClient, KeyPublisher
 
-reset_logs = os.getenv("AUGUR_RESET_LOGS", 'True').lower() in ('true', '1', 't', 'y', 'yes')
+reset_logs = SystemEnv.get_bool("AUGUR_RESET_LOGS", True)
 
 logger = SystemLogger("collectoss", reset_logfiles=reset_logs).get_logger()
 
@@ -61,7 +62,7 @@ def start(ctx, disable_collection, development, pidfile, port):
     signal.signal(signal.SIGINT, manager.shutdown_signal_handler)
 
     try:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1":
             raise_open_file_limit(100000)
     except Exception as e:
         logger.error(
@@ -71,10 +72,10 @@ def start(ctx, disable_collection, development, pidfile, port):
         raise e
 
     if development:
-        os.environ["AUGUR_DEV"] = "1"
+        SystemEnv.set("AUGUR_DEV", "1")
         logger.info("Starting in development mode")
 
-    os.environ["AUGUR_PIDFILE"] = pidfile
+    SystemEnv.set("AUGUR_PIDFILE", pidfile)
 
     try:
         gunicorn_location = os.getcwd() + "/collectoss/api/gunicorn_conf.py"
@@ -86,10 +87,10 @@ def start(ctx, disable_collection, development, pidfile, port):
     if not port:
         port = get_value("Server", "port")
 
-    os.environ["AUGUR_PORT"] = str(port)
+    SystemEnv.set("AUGUR_PORT", str(port))
 
     if disable_collection:
-        os.environ["AUGUR_DISABLE_COLLECTION"] = "1"
+        SystemEnv.set("AUGUR_DISABLE_COLLECTION", "1")
 
     core_worker_count = get_value("Celery", 'core_worker_count')
     secondary_worker_count = get_value("Celery", 'secondary_worker_count')
@@ -130,7 +131,7 @@ def start(ctx, disable_collection, development, pidfile, port):
     processes = start_celery_worker_processes((core_worker_count, secondary_worker_count, facade_worker_count), disable_collection)
     manager.processes = processes
 
-    celery_beat_schedule_db = os.getenv("CELERYBEAT_SCHEDULE_DB", "celerybeat-schedule.db")
+    celery_beat_schedule_db = SystemEnv.get("CELERYBEAT_SCHEDULE_DB", "celerybeat-schedule.db")
     if os.path.exists(celery_beat_schedule_db):
             logger.info("Deleting old task schedule")
             os.remove(celery_beat_schedule_db)
@@ -144,7 +145,7 @@ def start(ctx, disable_collection, development, pidfile, port):
     manager.keypub = keypub
 
     if not disable_collection:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1":
             orchestrator = subprocess.Popen("python keyman/Orchestrator.py".split())
 
         # Wait for orchestrator startup
@@ -355,10 +356,10 @@ def export_env(config):
     Exports your GitHub key and database credentials
     """
 
-    export_file = open(os.getenv('AUGUR_EXPORT_FILE', 'collectoss_export_env.sh'), 'w+')
+    export_file = open(SystemEnv.get('AUGUR_EXPORT_FILE') or 'collectoss_export_env.sh', 'w+')
     export_file.write('#!/bin/bash')
     export_file.write('\n')
-    env_file = open(os.getenv('AUGUR_ENV_FILE', 'docker_env.txt'), 'w+')
+    env_file = open(SystemEnv.get('AUGUR_ENV_FILE') or 'docker_env.txt', 'w+')
 
     for env_var in config.get_env_config().items():
         if "LOG" not in env_var[0]:
@@ -403,7 +404,7 @@ def get_backend_processes():
     for process in psutil.process_iter(['cmdline', 'name', 'environ']):
         if process.info['cmdline'] is not None and process.info['environ'] is not None:
             try:
-                if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in ''.join(process.info['cmdline'][:]).lower():
+                if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in ''.join(process.info['cmdline'][:]).lower():
                     if process.pid != os.getpid():
                         process_list.append(process)
             except (KeyError, FileNotFoundError):

collectoss/application/cli/collection.py

@@ -14,6 +14,7 @@
 import traceback
 import sqlalchemy as s
 
+from collectoss.application.environment import SystemEnv
 from collectoss.tasks.start_tasks import collection_monitor, create_collection_status_records
 from collectoss.tasks.git.facade_tasks import clone_repos
 from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
@@ -45,7 +46,7 @@ def start(ctx, development):
     """Start CollectOSS's backend server."""
 
     try:
-        if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
+        if SystemEnv.get('AUGUR_DOCKER_DEPLOY') != "1":
             raise_open_file_limit(100000)
     except Exception as e: 
         logger.error(
@@ -75,7 +76,7 @@ def start(ctx, development):
         keypub.publish(key, "gitlab_rest")
 
     if development:
-        os.environ["AUGUR_DEV"] = "1"
+        SystemEnv.set("AUGUR_DEV", "1")
         logger.info("Starting in development mode")
 
     core_worker_count = get_value("Celery", 'core_worker_count')
@@ -237,7 +238,7 @@ def get_collection_processes():
 def is_collection_process(process):
 
     command = ''.join(process.info['cmdline'][:]).lower()
-    if os.getenv('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
+    if SystemEnv.get('VIRTUAL_ENV') in process.info['environ']['VIRTUAL_ENV'] and 'python' in command:
         if process.pid != os.getpid():
 
             if "collectossbackendcollection" in command  or "celery_app.celery_appbeat" in command:

collectoss/application/cli/db.py

@@ -28,6 +28,7 @@
     process_repo_csv,
     process_repo_group_csv,
 )
+from collectoss.application.environment import SystemEnv
 
 logger = logging.getLogger(__name__)
 
@@ -379,7 +380,7 @@ def get_api_key(ctx):
     short_help="Check the ~/.pgpass file for CollectOSS's database credentials",
 )
 def check_pgpass():
-    db_environment_var = getenv("AUGUR_DB")
+    db_environment_var = SystemEnv.get("AUGUR_DB")
     if db_environment_var:
         # gets the user, passowrd, host, port, and database_name out of environment variable
         # assumes database string of structure <beginning_of_db_string>//<user>:<password>@<host>:<port>/<database_name>
@@ -495,7 +496,7 @@ def run_psql_command_in_database(target_type, target):
         logger.error("Invalid target type. Exiting...")
         exit(1)
 
-    db_environment_var = getenv("AUGUR_DB")
+    db_environment_var = SystemEnv.get("AUGUR_DB")
 
     # db_json_file_location = os.getcwd() + "/db.config.json"
     # db_json_exists = os.path.exists(db_json_file_location)

collectoss/application/config.py

@@ -7,6 +7,8 @@
 from collectoss.application.db.models import Config 
 from collectoss.application.db.util import execute_session_query, convert_type_of_value
 from pathlib import Path
+from collectoss.application.environment import SystemEnv
+
 import logging
 
 def get_development_flag_from_config():
@@ -27,7 +29,7 @@ def get_development_flag_from_config():
     return flag
 
 def get_development_flag():
-    return os.getenv("AUGUR_DEV") or get_development_flag_from_config() or False
+    return SystemEnv.get("AUGUR_DEV") or get_development_flag_from_config() or False
 
 def redact_setting_value(section_name, setting_name, value):
     value_redacted = value if section_name != "Keys" else "REDACTED"
@@ -167,7 +169,7 @@ def __init__(self, logger, session: DatabaseSession, config_sources: list = None
                 JsonConfig(default_config, logger)
             ]
 
-            config_dir = Path(os.getenv("CONFIG_DATADIR", "./"))
+            config_dir = Path(SystemEnv.get("CONFIG_DATADIR") or "./")
             config_path = config_dir.joinpath("augur.json")
             if config_path.exists():
                 config_sources.append(JsonConfig(json.loads(config_path.read_text(encoding="UTF-8")), logger))