698.workflow install tool

fre/fre.py

@@ -22,7 +22,9 @@
 # click and lazy group loading
 @click.group(
     cls = LazyGroup,
-    lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
+
+    lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli",
+                       "pp": ".pp.frepp.pp_cli",
                        "catalog": ".catalog.frecatalog.catalog_cli",
                        "list": ".list_.frelist.list_cli",
                        "check": ".check.frecheck.check_cli",

fre/workflow/__init__.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+def make_workflow_name(experiment : Optional[str] = None) -> str:
+    """
+    Function that takes in a triplet of tags for a model experiment, platform, and target, and
+    returns a directory name for the corresponding pp workflow. Because this is often given by
+    user to the shell being used by python, we split/reform the string to remove semi-colons or
+    spaces that may be used to execute an arbitrary command with elevated privileges.
+
+    :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None
+    :type experiment: str
+    :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None
+    :type platform: str
+    :param target: Options used for the model compiler (e.g. prod-openmp), default None
+    :type target: str
+    :return: string created in specific format from the input strings
+    :rtype: str
+
+    .. note:: if any arguments are None, then "None" will appear in the workflow name
+    """
+    name = f'{experiment}__{platform}__{target}'
+    return ''.join(
+                      (''.join(
+                          name.split(' ')
+                              )
+                      ).split(';')
+                  ) # user-input sanitation, prevents some malicious cmds from being executed with privileges

fre/workflow/freworkflow.py

@@ -0,0 +1,39 @@
+''' fre workflow '''
+import os
+import click
+import logging
+fre_logger = logging.getLogger(__name__)
+
+#fre tools
+#from . import checkout_script
+from . import install_script
+#from . import run_script
+
+@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210)))
+def workflow_cli():
+    ''' entry point to fre workflow click commands '''
+
+@workflow_cli.command()
+@click.option("-e", "--experiment",
+              type=str,
+              required=True,
+              help="Experiment name")
+@click.option("--src-dir",
+              type=str,
+              envvar="TMPDIR",
+              default=os.path.expanduser("~/.fre"),
+              required=True,
+              help="Path to cylc-src directory")
+@click.option("--target-dir",
+              type=str,
+              help="""Target directory to install the cylc
+                      workflow into. Default location is
+                      ~/cylc-run/<workflow name>""")
+@click.option("--force-install",
+              type=bool,
+              help="If cylc-run/[workflow_name] exists")
+def install(experiment, src_dir, target_dir):
+    """
+    Install workflow configuration
+    """
+    install_script.workflow_install(experiment, src_dir, target_dir)

fre/workflow/install_script.py

@@ -0,0 +1,73 @@
+''' fre workflow install '''
+from pathlib import Path
+import subprocess
+import logging
+from fre.app.helpers import change_directory
+
+fre_logger =logging.getLogger(__name__)
+
+def workflow_install(experiment: str, src_dir: str, target_dir: str, force_install: bool):
+    """
+    Install the Cylc workflow definition located in
+    cylc-src/$(experiment) to cylc-run/$(experiment)
+
+    :param experiment: Experiment names from the yaml displayed by
+                       fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip);
+                       default None
+    :type experiment: str
+    :param src_dir: src_dir/workflow_name
+    :type src_dir:
+    :param target_dir:
+    :type target_dir:
+    """
+    workflow_name = experiment
+
+    # Check src_dir exists
+    if not Path(src_dir):
+        raise ValueError("""Cylc source directory ({src_dir}) could not be found! Try specifying
+                            path by passing --src-dir [path].""")
+
+    # if the cylc-run directory already exists,
+    # then check whether the cylc expanded definition (cylc config)
+    # is identical. If the same, good. If not, bad.
+    install_dir = Path(f"{target_dir}/cylc-run")
+    if Path(install_dir).is_dir():
+        fre_logger.warning(" *** PREVIOUS INSTALL FOUND: %s ***", install_dir)
+        if force_install:
+            fre_logger.warning(" *** REMOVING %s *** ", install_dir)
+            install_output = subprocess.run(["cylc", "clean", f"{install_dir}/{workflow_name}"],
+                                            capture_output = True,
+                                            text = True,
+                                            check = True)
+            fre_logger.debug(install_output)
+        else:
+            # must convert from bytes to string for proper comparison
+            installed_def = subprocess.run(["cylc", "config", workflow_name],
+                                           capture_output=True,
+                                           check=True).stdout.decode('utf-8')
+            with change_directory(src_dir):
+                source_def = subprocess.run(['cylc', 'config', '.'],
+                                            capture_output=True,
+                                            check=True).stdout.decode('utf-8')
+
+            if installed_def == source_def:
+                fre_logger.warning("""NOTE: Workflow '%s/%s}' already ",
+                                      installed, and the definition is unchanged""", install_dir, workflow_name)
+            else:
+                raise ValueError(f"""ERROR: Workflow '{install_dir}/{workflow_name}' already
+                                     installed, and the definition has changed!
+
+                                     Please remove and re-install the workflow with one of these options:
+                                       - fre workflow install -e {experiment} --src-dir {src_dir} --target-dir {target_dir} --force-install"
+                                       - cylc clean {install_dir}/{workflow_name}, then re-run install command""")
+
+    if not Path(install_dir).is_dir():
+        fre_logger.warning("NOTE: About to install workflow into ~/cylc-run/%s", workflow_name)
+        if not target_dir:
+            # install workflow in default home location
+            cmd = f"cylc install --no-run-name {src_dir}"
+            subprocess.run(cmd, shell=True, check=True)
+        else:
+            # symlink the workflow and associated files in target_dir
+            cmd = f"cylc install --no-run-name {src_dir} --symlink-dirs='{target_dir}/{workflow_name}'"
+            subprocess.run(cmd, shell=True, check=True)

fre/workflow/tests/AM5_example/am5.yaml

@@ -0,0 +1,68 @@
+# reusable variables
+fre_properties:
+  - &AM5_VERSION        "am5f7b12r1"
+  - &FRE_STEM           !join [am5/, *AM5_VERSION]
+
+  # amip
+  - &EXP_AMIP_START     "19790101T0000Z"
+  - &EXP_AMIP_END       "20200101T0000Z"
+  - &ANA_AMIP_START     "19800101T0000Z"
+  - &ANA_AMIP_END       "20200101T0000Z"
+
+  - &PP_AMIP_CHUNK96    "P1Y"
+  - &PP_AMIP_CHUNK384   "P1Y"
+  - &PP_XYINTERP96      "180,288"
+  - &PP_XYINTERP384     "720,1152"
+
+  # climo
+  - &EXP_CLIMO_START96  "0001"
+  - &EXP_CLIMO_END96    "0011"
+  - &ANA_CLIMO_START96  "0002"
+  - &ANA_CLIMO_END96    "0011"
+
+  - &EXP_CLIMO_START384 "0001"
+  - &EXP_CLIMO_END384   "0006"
+  - &ANA_CLIMO_START384 "0002"
+  - &ANA_CLIMO_END384   "0006"
+
+  # coupled
+  - &PP_CPLD_CHUNK_A    "P5Y"
+  - &PP_CPLD_CHUNK_B    "P20Y"
+
+  # grids
+  - &GRID_SPEC96         "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar"
+
+  # compile information
+  - &release             "f1a1r1"
+  - &INTEL               "intel-classic"
+  - &FMSincludes         "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+  - &momIncludes         "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
+
+# compile information
+build:
+  compileYaml: "compile.yaml"
+  platformYaml: "yaml_include/platforms.yaml"
+
+experiments:
+  - name: "c96L65_am5f7b12r1_amip_TESTING"
+    settings: "yaml_include/settings.yaml"
+    pp:
+      - "yaml_include/pp.c96_amip.yaml"
+      - "yaml_include/pp-test.c96_amip.yaml"
+  - name: "c96L65_am5f7b12r1_amip_TESTING_WRONG"
+    settings: "yaml_include/settings_WRONG.yaml"
+    pp:
+      - "yaml_include/pp.c96_amip.yaml"
+
+#  amip:
+#    settings:
+#      - shared/settings.yaml
+#      - shared/directories.yaml
+#    run:
+#      version: 1.1
+#      - run/inputs.yaml
+#      - run/runtime.yaml
+#    postprocess:
+#      version: 2.0
+#      - pp/components
+#      - analysis/legacy-bw.yaml

fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml

@@ -0,0 +1,33 @@
+# local reusable variable overrides
+fre_properties:
+  - &custom_interp "200,200"
+
+#c96_amip_postprocess:
+postprocess:
+  components:
+    - type: "atmos_cmip-TEST"
+      sources:
+        - history_file: "atmos_month_cmip"
+        - history_file: "atmos_8xdaily_cmip"
+        - history_file: "atmos_daily_cmip"
+      sourceGrid: "cubedsphere"
+      xyInterp: *custom_interp
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos-TEST"
+      sources:
+        - history_file: "atmos_month"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos_level_cmip-TEST"
+      sources:
+        - history_file: "atmos_level_cmip"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False

fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml

@@ -0,0 +1,94 @@
+# local reusable variable overrides
+fre_properties: 
+  - &custom_interp "180,360"
+
+#c96_amip_postprocess:
+postprocess:
+  # main pp instructions
+  components:
+    - type: "atmos_cmip"
+      sources:
+        - history_file: "atmos_month_cmip" 
+        - history_file: "atmos_8xdaily_cmip" 
+        - history_file: "atmos_daily_cmip"
+      sourceGrid: "cubedsphere"
+      xyInterp: *custom_interp
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos"
+      sources:
+        - history_file: "atmos_month"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: True
+    - type: "atmos_level_cmip"
+      sources:
+        - history_file: "atmos_level_cmip"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos_level"
+      sources:
+        - history_file: "atmos_month"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos_month_aer"
+      sources:
+        - history_file: "atmos_month_aer"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order1"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos_diurnal"
+      sources:
+        - history_file: "atmos_diurnal"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos_scalar"
+      sources:
+        - history_file: "atmos_scalar"
+      postprocess_on: True
+    - type: "aerosol_cmip"
+      xyInterp: *PP_XYINTERP96
+      sources:
+        - history_file: "aerosol_month_cmip"
+      sourceGrid: "cubedsphere"
+      interpMethod: "conserve_order1"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "land"
+      sources:
+        - history_file: "land_month"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order1"
+      inputRealm: 'land'
+      postprocess_on: False
+    - type: "land_cmip"
+      sources:
+        - history_file: "land_month_cmip"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order1"
+      inputRealm: 'land'
+      postprocess_on: False
+    - type: "tracer_level"
+      sources:
+        - history_file: "atmos_tracer"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order1"
+      inputRealm: 'atmos'
+      postprocess_on: False

fre/workflow/tests/AM5_example/yaml_include/settings.yaml

@@ -0,0 +1,32 @@
+#workflow repositories
+workflow:
+  run_workflow:
+    repo: "https://github.com/NOAA-GFDL"
+    version: "tbd"
+  pp_workflow:
+    repo: "https://github.com/NOAA-GFDL/fre-workflows.git"
+    version: "main"
+
+#c96_amip_directories:
+directories:
+  history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
+  pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
+  analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
+  ptmp_dir:     "/xtmp/$USER/ptmp"
+
+#c96_amip_postprocess:
+postprocess:
+  settings:
+    history_segment:    "P1Y"
+    site:               "ppan"
+    pp_start:   *ANA_AMIP_START
+    pp_stop:    *ANA_AMIP_END
+    pp_chunks:  [*PP_AMIP_CHUNK96]
+    pp_grid_spec: *GRID_SPEC96
+  switches:
+    clean_work:         True
+    do_refinediag:      False
+    do_atmos_plevel_masking: True
+    do_preanalysis:     False
+    do_analysis:        True
+    do_analysis_only:   False