Auto rebuild (#435)

* Auto rebuild

* Upd changelog

Author: joente

CHANGELOG.md

@@ -1,3 +1,7 @@
+# v1.8.4-alpha0
+
+* Added `--auto-rebuild` flag, pr #435.
+
 # v1.8.3
 
 * Added the `America/Coyhaique` time zone, pr #432.

inc/ti/args.h

@@ -20,6 +20,7 @@ struct ti_args_s
     int32_t init;
     int32_t log_colorized;
     int32_t rebuild;
+    int32_t auto_rebuild;
     int32_t forget_nodes;
     int32_t yes;
     int32_t version;

inc/ti/version.h

@@ -6,7 +6,7 @@
 
 #define TI_VERSION_MAJOR 1
 #define TI_VERSION_MINOR 8
-#define TI_VERSION_PATCH 3
+#define TI_VERSION_PATCH 4
 
 /* The syntax version is used to test compatibility with functions
  * using the `ti_nodes_check_syntax()` function */
@@ -25,7 +25,7 @@
  *  "-rc0"
  *  ""
  */
-#define TI_VERSION_PRE_RELEASE ""
+#define TI_VERSION_PRE_RELEASE "-alpha0"
 
 #define TI_MAINTAINER \
     "Jeroen van der Heijden <jeroen@cesbit.com>"

itest/lib/node.py

@@ -98,8 +98,8 @@ async def wait_join(self, secret: str, timeout: int = 5):
         await self.expect(
             'start listening for node connections', timeout=timeout)
 
-    async def run(self, timeout: int = 5):
-        self.start()
+    async def run(self, timeout: int = 5, auto_rebuild: bool = False):
+        self.start(auto_rebuild=auto_rebuild)
         await self.expect(
             'start listening for node connections', timeout=timeout)
 
@@ -245,7 +245,7 @@ def version(self):
                 self.n == THINGSDB_NODE_OUTPUT):
             print(Color.node(self.n, m.group(1)))
 
-    def start(self, init=None, secret=None):
+    def start(self, init=None, secret=None, auto_rebuild=False):
         self.queue = asyncio.Queue()
 
         command = THINGSDB_MEMCHECK + [
@@ -260,6 +260,9 @@ def start(self, init=None, secret=None):
         elif secret:
             command.extend(['--secret', secret])
 
+        if auto_rebuild:
+            command.append('--auto-rebuild')
+
         r, w = os.pipe()
         w = os.fdopen(w, 'w')
 

itest/run_all_tests.py

@@ -26,6 +26,7 @@
 from test_nodes import TestNodes
 from test_operators import TestOperators
 from test_procedures import TestProcedures
+from test_recovery import TestRecovery
 from test_relations import TestRelations
 from test_restriction import TestRestriction
 from test_room import TestRoom
@@ -101,6 +102,7 @@ def no_mem_test(test_class):
     run_test(TestNodes(), hide_version=hide_version())
     run_test(TestOperators(), hide_version=hide_version())
     run_test(TestProcedures(), hide_version=hide_version())
+    run_test(TestRecovery(), hide_version=hide_version())
     run_test(TestRelations(), hide_version=hide_version())
     run_test(TestRestriction(), hide_version=hide_version())
     run_test(TestRoom(), hide_version=hide_version())

itest/test_recovery.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+import asyncio
+import os
+from lib import run_test
+from lib import default_test_setup
+from lib.testbase import TestBase
+from lib.client import get_client
+from thingsdb.exceptions import OperationError
+
+
+script_dir = os.path.dirname(__file__)
+
+
+class TestRecovery(TestBase):
+
+    title = 'Test auto recovery'
+
+    @default_test_setup(num_nodes=3, seed=2)
+    async def async_run(self):
+        await self.node0.init_and_run()
+
+        cl0 = await get_client(self.node0)
+        cl0.set_default_scope('//stuff')
+
+        await self.node1.join_until_ready(cl0)
+        await self.node2.join_until_ready(cl0)
+
+        cl1 = await get_client(self.node1)
+        cl2 = await get_client(self.node2)
+
+        for i, c in enumerate('abcdefghijklmnop'):
+            await cl0.query("""//ti
+                      .set(c, i);
+            """, i=i, c=c)
+
+        await asyncio.sleep(1.0)
+
+        await self.node0.shutdown()
+        fn = os.path.join(script_dir,
+                          'testdir',
+                          'tdb0',
+                          'store',
+                          '00000000002',
+                          'gcprops.mp')
+        with open(fn, 'wb') as fp:
+            fp.write(b'<<>>')  # force corruption
+
+        await self.node0.run(auto_rebuild=True)
+
+        cl0 = await get_client(self.node0)
+
+        for cl in (cl0, cl1, cl2):
+            res = await cl.query('.d;', scope='//stuff')
+            self.assertEqual(res, 3)
+
+        await cl0.close_and_wait()
+        await cl1.close_and_wait()
+        await cl2.close_and_wait()
+
+
+if __name__ == '__main__':
+    run_test(TestRecovery())

main.c

@@ -254,8 +254,26 @@ int main(int argc, char * argv[])
         }
         else if ((rc = ti_store_restore()))
         {
-            printf("error loading ThingsDB\n");
-            goto stop;
+            if (ti.args->auto_rebuild && ti.nodes->vec->n >= 2)
+            {
+                printf(
+                    "*********************************\n"
+                    "  Failed to load ThingsDB\n"
+                    "  Starting Auto Rebuild ...\n"
+                    "*********************************\n");
+                if ((rc = ti_rebuild()))
+                {
+                    printf("error initiating auto rebuild\n");
+                    goto stop;
+                }
+            }
+            else
+            {
+                printf(
+                    "error loading ThingsDB; "
+                    "you might want to try --rebuild or --auto-rebuild\n");
+                goto stop;
+            }
         }
     }
     else

src/ti/archive.c

@@ -1,6 +1,8 @@
 /*
  * ti/archive.c
  */
+#define _GNU_SOURCE
+#include <dirent.h>
 #include <assert.h>
 #include <ctype.h>
 #include <stdlib.h>

src/ti/args.c

@@ -29,6 +29,7 @@ int ti_args_create(void)
     args->init = 0;
     args->log_colorized = 0;
     args->rebuild = 0;
+    args->auto_rebuild = 0;
     args->forget_nodes = 0;
     args->version = 0;
 
@@ -128,6 +129,18 @@ int ti_args_parse(int argc, char *argv[])
         .choices = NULL,
     };
 
+    argparse_argument_t auto_rebuild_ = {
+        .name = "auto-rebuild",
+        .shortcut = 0,
+        .help = "auto rebuild this node on failed (can only be used when having >1 nodes)",
+        .action = ARGPARSE_STORE_TRUE,
+        .default_int32_t = 0,
+        .pt_value_int32_t = &args->auto_rebuild,
+        .str_default = NULL,
+        .str_value = NULL,
+        .choices = NULL,
+    };
+
     argparse_argument_t forget_nodes_ = {
         .name = "forget-nodes",
         .shortcut = 0,
@@ -194,6 +207,7 @@ int ti_args_parse(int argc, char *argv[])
             argparse_add_argument(parser, &force_) ||
             argparse_add_argument(parser, &secret_) ||
             argparse_add_argument(parser, &rebuild_) ||
+            argparse_add_argument(parser, &auto_rebuild_) ||
             argparse_add_argument(parser, &forget_nodes_) ||
             argparse_add_argument(parser, &yes_) ||
             argparse_add_argument(parser, &version_) ||
@@ -283,7 +297,6 @@ int ti_args_parse(int argc, char *argv[])
                 rc = -1;
             }
         }
-
     }
 
     if (parser->show_help)

src/ti/evars.c

@@ -93,6 +93,9 @@ void ti_evars_arg_parse(void)
     evars__bool_arg(
             "THINGSDB_DEPLOY",
             &ti.args->deploy);
+    evars__bool_arg(
+            "THINGSDB_AUTO_REBUILD",
+            &ti.args->auto_rebuild);
     evars__str_arg(
             "THINGSDB_SECRET",
             ti.args->secret);