Merge pull request #67 from d4rkstar/minio_new

sciabarracom · web-flow · commit 07f3600dd906 · 2025-07-12T21:16:01.000+02:00
Minio new + remove auto from apihost + set default schema for databases
diff --git a/Dockerfile b/Dockerfile
@@ -137,7 +137,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone &
     curl -sL "$WSK_BASE/$WSK_VERSION/OpenWhisk_CLI-$WSK_VERSION-linux-$ARCH.tgz" | tar xzvf - -C /usr/bin/ && \
     # install minio
     MINIO_BASE=https://dl.min.io/client/mc/release/linux && \
-    MC_VER=RELEASE.2023-03-23T20-03-04Z && \
+    MC_VER=RELEASE.2025-05-21T01-59-54Z && \
     curl -sL "$MINIO_BASE-$ARCH/archive/mc.${MC_VER}" -o /usr/bin/mc && chmod +x /usr/bin/mc && \
     # install taskfile
     curl -sL https://taskfile.dev/install.sh | sh -s -- -d -b /usr/bin
diff --git a/nuvolaris/apihost_util.py b/nuvolaris/apihost_util.py
@@ -161,17 +161,6 @@ def extract_port(url):
     parsed_url = urllib.parse.urlparse(url)
     return parsed_url.port
 
-def extract_hostname(url):
-    """
-    Parse a url and extract the hostname part
-    >>> extract_hostname('http://localhost:8080')
-    'localhost'
-    >>> extract_hostname('https://nuvolaris.org')    
-    'nuvolaris.org'
-    """
-    parsed_url = urllib.parse.urlparse(url)
-    return parsed_url.hostname
-
 def split_hostname_port(url):
     """
     Parse a url and extract the port part
@@ -190,9 +179,6 @@ def get_user_static_hostname(runtime, username, apihost):
     inside the cm/config configMap prepending the user_namespace when needed.
     """
     
-    if apihost not in ["auto"]:
-        return apihost
-
     apihost_url = util.get_apihost_from_config_map()
 
     if apihost_url:
diff --git a/nuvolaris/kube.py b/nuvolaris/kube.py
@@ -37,7 +37,7 @@
 # default output is text
 # if you specify jsonpath it will filter and parse the json output
 # returns exceptions if errors
-def kubectl(*args, namespace="nuvolaris", input=None, jsonpath=None, debugresult=True):
+def kubectl(*args, namespace="nuvolaris", input=None, jsonpath=None, debugresult=True, timeout=None):
     # support for mocked requests
     mres = mocker.invoke(*args)
     if mres:
@@ -55,7 +55,7 @@ def kubectl(*args, namespace="nuvolaris", input=None, jsonpath=None, debugresult
         
     # executing
     logging.debug(cmd)
-    res = subprocess.run(cmd, capture_output=True, input=input)
+    res = subprocess.run(cmd, capture_output=True, input=input, timeout=timeout)
 
     global returncode, output, error
     returncode = res.returncode
diff --git a/nuvolaris/minio_util.py b/nuvolaris/minio_util.py
@@ -54,14 +54,23 @@ def mc(self, *kwargs):
             error = res.stderr.decode()
             
             if returncode != 0:
+                self.last_output = error
                 logging.error(error)
             else:
+                self.last_output = output
                 logging.info(output)
 
             return returncode == 0
         except Exception as e:
-                        logging.error(e)
-                        return e
+                    logging.error(e)
+                    return e
+
+    def get_last_output(self):
+        """
+        returns the last output of the executed command
+        """
+        return self.last_output if hasattr(self, 'last_output') else None    
+    
 
     def add_user(self, username, secret_key):
         """
@@ -101,6 +110,7 @@ def assign_quota_to_bucket(self, bucket_name, quota):
         assign the specified quota on the given bucket
         """        
         return util.check(self.mc("quota","set",f"{self.alias}/{bucket_name}","--size", f"{quota}m"),"assign_quota_to_bucket",True)
+        
 
     def assign_policy_to_user(self, username, policy):
         """
diff --git a/nuvolaris/policies/minio_rw_policy_tpl.json b/nuvolaris/policies/minio_rw_policy_tpl.json
@@ -7,12 +7,7 @@
       ],
       "Effect": "Allow",
       "Resource": [
-        {% for bucket_arn in bucket_arns -%}
-          "arn:aws:s3:::{{bucket_arn}}"
-          {% if not loop.last %}
-          ,
-          {% endif %}
-        {% endfor %}
+        {% for bucket_arn in bucket_arns %}"arn:aws:s3:::{{ bucket_arn }}"{{ "," if not loop.last }}{% endfor %}
       ]
     }
   ]
diff --git a/nuvolaris/postgres_operator.py b/nuvolaris/postgres_operator.py
@@ -178,12 +178,16 @@ def render_postgres_script(namespace,template,data):
     file = ntp.spool_template(template, out, data)
     return os.path.abspath(file)
 
-def exec_psql_command(pod_name,path_to_psql_script,path_to_pgpass):
+def exec_psql_command(pod_name,path_to_psql_script,path_to_pgpass,additional_psql_args=''):
     logging.info(f"passing script {path_to_psql_script} to pod {pod_name}")
     res = kube.kubectl("cp",path_to_psql_script,f"{pod_name}:{path_to_psql_script}")
     res = kube.kubectl("cp",path_to_pgpass,f"{pod_name}:/tmp/.pgpass")
     res = kube.kubectl("exec","-it",pod_name,"--","/bin/bash","-c",f"chmod 600 /tmp/.pgpass")
-    res = kube.kubectl("exec","-it",pod_name,"--","/bin/bash","-c",f"PGPASSFILE='/tmp/.pgpass' psql --username postgres --dbname postgres -f {path_to_psql_script}")
+
+    cmd = f"PGPASSFILE='/tmp/.pgpass' psql --username postgres --dbname postgres {additional_psql_args} -f {path_to_psql_script}"
+    logging.info(f"executing command: {cmd}")
+    res = kube.kubectl("exec","-it",pod_name,"--","/bin/bash","-c",cmd)
+    
     os.remove(path_to_psql_script)
     os.remove(path_to_pgpass)
     return res
@@ -209,6 +213,10 @@ def create_db_user(ucfg: UserConfig, user_metadata: UserMetadata):
             if res:
                 _add_pdb_user_metadata(ucfg, user_metadata)
                 
+                path_to_pgpass = render_postgres_script(ucfg.get('namespace'),"dbname_pgpass_tpl.properties",data)
+                path_to_schema_script = render_postgres_script(ucfg.get('namespace'),"postgres_manage_user_schema_tpl.sql",data)
+                res = exec_psql_command_in_db(database,pod_name,path_to_schema_script,path_to_pgpass)
+
                 data["extensions"]=["vector"]
                 path_to_pgpass = render_postgres_script(ucfg.get('namespace'),"dbname_pgpass_tpl.properties",data)
                 path_to_extensions_script = render_postgres_script(ucfg.get('namespace'),"postgres_manage_user_extension_tpl.sql",data)
@@ -232,12 +240,17 @@ def delete_db_user(namespace, database):
         data["database"]=database
         data["mode"]="delete"
 
-        path_to_pgpass = render_postgres_script(namespace,"pgpass_tpl.properties",data)
-        path_to_mdb_script = render_postgres_script(namespace,"postgres_manage_user_tpl.sql",data)
+        
         pod_name = util.get_pod_name_by_selector("app=nuvolaris-postgres","{.items[?(@.metadata.labels.replicationRole == 'primary')].metadata.name}")
 
         if(pod_name):
-            res = exec_psql_command(pod_name,path_to_mdb_script,path_to_pgpass)
+            path_to_pgpass = render_postgres_script(namespace,"pgpass_tpl.properties",data)
+            path_to_ter_script = render_postgres_script(namespace,"postgres_terminate_tpl.sql",data)    
+            res = exec_psql_command(pod_name,path_to_ter_script,path_to_pgpass,' -q -t ')
+
+            path_to_pgpass = render_postgres_script(namespace,"pgpass_tpl.properties",data)
+            path_to_mdb_script = render_postgres_script(namespace,"postgres_manage_user_tpl.sql",data)    
+            res += exec_psql_command(pod_name,path_to_mdb_script,path_to_pgpass)
             return res 
 
         return None
diff --git a/nuvolaris/templates/01-minio-dep.yaml b/nuvolaris/templates/01-minio-dep.yaml
@@ -43,7 +43,7 @@ spec:
       {% endif %}      
       containers:
       - name: minio
-        image: bitnami/minio:2023.3.24
+        image: bitnami/minio:2025.6.13
         securityContext:
           capabilities:
             drop:
@@ -53,11 +53,7 @@ spec:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: false
           runAsNonRoot: true                  
-        command:
-        - /bin/bash
-        - -c
-        args: 
-        - minio server /data --console-address :9090
+        command: ["/bin/bash", "-c", "minio server /data --console-address :9090"]
         env:
         - name: MINIO_ROOT_USER
           value: {{minio_root_user}}
diff --git a/nuvolaris/templates/postgres_manage_user_extension_tpl.sql b/nuvolaris/templates/postgres_manage_user_extension_tpl.sql
@@ -19,7 +19,7 @@
 
 {% if mode == 'create' %}
 {% for extension in extensions -%}
-CREATE EXTENSION IF NOT EXISTS {{extension}};
+CREATE EXTENSION IF NOT EXISTS {{extension}} WITH SCHEMA {{username}}_schema;
 {% endfor %}
 {% endif %}
 
diff --git a/nuvolaris/templates/postgres_manage_user_schema_tpl.sql b/nuvolaris/templates/postgres_manage_user_schema_tpl.sql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+{% if mode == 'create' %}
+-- Create schema only if not exists and owned by correct user
+CREATE SCHEMA IF NOT EXISTS {{username}}_schema;
+ALTER SCHEMA {{username}}_schema OWNER TO {{username}};
+ALTER DATABASE {{database}} SET search_path TO {{username}}_schema, pg_catalog;
+{% endif %}
diff --git a/nuvolaris/templates/postgres_manage_user_tpl.sql b/nuvolaris/templates/postgres_manage_user_tpl.sql
@@ -25,12 +25,7 @@ REVOKE CONNECT ON DATABASE {{database}} from public;
 {% endif %}
 
 {% if mode == 'delete' %}
-SELECT pg_terminate_backend(pg_stat_activity.pid)
-FROM pg_stat_activity
-WHERE pg_stat_activity.datname = '{{database}}';
-
 DROP DATABASE {{database}};
-
 DROP OWNED BY {{username}};
 DROP USER {{username}};
 {% endif %}
diff --git a/nuvolaris/templates/postgres_terminate_tpl.sql b/nuvolaris/templates/postgres_terminate_tpl.sql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DO $$
+BEGIN
+  PERFORM pg_catalog.pg_terminate_backend(pid)
+  FROM pg_catalog.pg_stat_activity
+  WHERE pg_stat_activity.datname = '{{database}}'
+    AND pg_stat_activity.pid <> pg_catalog.pg_backend_pid();
+END;
+$$;
+
+
diff --git a/nuvolaris/testutil.py b/nuvolaris/testutil.py
@@ -311,3 +311,11 @@ def load_sample_runtimes(name="runtimes"):
 
 def enable_debug_logging():
     logging.basicConfig(level=logging.DEBUG)
+
+def run_proc(cmd):
+    try:
+        get_ipython().system(cmd)
+    except NameError:
+        import subprocess
+        subprocess.run(cmd.split(), check=True)
+        
diff --git a/tests/kind/minio_quota_test.ipy b/tests/kind/minio_quota_test.ipy
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os
+
+import nuvolaris.config as cfg
+import nuvolaris.minio_deploy as minio
+import nuvolaris.minio_util as mutil
+import nuvolaris.testutil as tu
+import nuvolaris.util as util
+import json
+
+tu.run_proc("kubectl -n nuvolaris delete all --all")
+tu.run_proc("kubectl -n nuvolaris delete pvc --all")
+
+# test
+assert(cfg.configure(tu.load_sample_config()))
+assert(cfg.detect_storage()["nuvolaris.storageclass"])
+
+os.environ['MINIO_API_HOST']='localhost'
+assert(minio.create())
+
+pod_name = util.get_pod_name("{.items[?(@.metadata.labels.app == 'minio')].metadata.name}")
+assert(pod_name)
+
+minioClient = mutil.MinioClient()
+assert(minioClient.make_bucket("ftt-data"))
+assert(minioClient.make_public_bucket("ftt-web"))
+assert(minioClient.add_user("ftt","jgfkjsgcasgfjgdsafgsdkfgkaj"))
+assert(minioClient.assign_rw_bucket_policy_to_user("ftt",["ftt-web/*","ftt-data/*"]))
+
+assert(minioClient.assign_quota_to_bucket("ftt-data",10))
+
+# upload 7M
+tu.run_proc(f"dd if=/dev/urandom of=/tmp/__minio_test_file1 bs=1M count=7")
+minioClient.mc("cp", f"/tmp/__minio_test_file1", f"local_ftt/ftt-data/file1")
+os.remove(f"/tmp/__minio_test_file1")
+
+# check quota is 7M
+assert(minioClient.mc("du", "local/ftt-data", "--json"))
+last_output = minioClient.get_last_output()
+quota_info = json.loads(last_output)
+size = quota_info.get('size', 0)
+assert(size == 7340032)  
+
+
+tu.run_proc(f"dd if=/dev/urandom of=/tmp/__minio_test_file2 bs=1M count=15")
+res = minioClient.mc("cp", f"/tmp/__minio_test_file2", f"local_ftt/ftt-data/file2")
+assert (res is False)
+last_output = minioClient.get_last_output()
+os.remove(f"/tmp/__minio_test_file2")
+assert(last_output.index("Bucket quota exceeded") >= 0)
+
+
+assert(minio.delete())
diff --git a/tests/kind/minio_static_test.ipy b/tests/kind/minio_static_test.ipy
@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-!kubectl -n nuvolaris delete all --all
-!kubectl -n nuvolaris delete pvc --all
 
 import os
 
@@ -25,6 +23,10 @@ import nuvolaris.minio_deploy as minio
 import nuvolaris.storage_static as static
 import nuvolaris.testutil as tu
 
+
+tu.run_proc("kubectl -n nuvolaris delete all --all")
+tu.run_proc("kubectl -n nuvolaris delete pvc --all")
+
 # test
 assert(cfg.configure(tu.load_sample_config()))
 assert(cfg.detect_storage()["nuvolaris.storageclass"])
diff --git a/tests/kind/minio_test.ipy b/tests/kind/minio_test.ipy
@@ -15,9 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-!kubectl -n nuvolaris delete all --all
-!kubectl -n nuvolaris delete pvc --all
-
 import os
 
 import nuvolaris.config as cfg
@@ -26,6 +23,9 @@ import nuvolaris.minio_deploy as minio
 import nuvolaris.testutil as tu
 import nuvolaris.util as util
 
+tu.run_proc("kubectl -n nuvolaris delete all --all")
+tu.run_proc("kubectl -n nuvolaris delete pvc --all")
+
 # test
 assert(cfg.configure(tu.load_sample_config()))
 assert(cfg.detect_storage()["nuvolaris.storageclass"])
diff --git a/tests/kind/minio_util_test.ipy b/tests/kind/minio_util_test.ipy

Original file line number	Diff line number	Diff line change
`@@ -7,12 +7,7 @@`
`7`	`7`	`],`
`8`	`8`	`"Effect": "Allow",`
`9`	`9`	`"Resource": [`
`10`		`- {% for bucket_arn in bucket_arns -%}`
`11`		`- "arn:aws:s3:::{{bucket_arn}}"`
`12`		`- {% if not loop.last %}`
`13`		`- ,`
`14`		`- {% endif %}`
`15`		`- {% endfor %}`
	`10`	`+ {% for bucket_arn in bucket_arns %}"arn:aws:s3:::{{ bucket_arn }}"{{ "," if not loop.last }}{% endfor %}`
`16`	`11`	`]`
`17`	`12`	`}`
`18`	`13`	`]`