[tools/shoestring]: add recovery tool

problem: There is no way to recover from lock files in the data folder
solution: create a docker-compose-recovery.yaml which will run the recovery tool

Author: Wayonb

tools/shoestring/README.md

@@ -401,3 +401,22 @@ cd symbol-product-directory/tools/shoestring
 python3 -m pip install -r requirements.txt
 PYTHONPATH=. python3 -m shoestring --help
 ```
+
+# Troubleshooting
+
+## Server or Broker failed to start due to lock files
+
+The docker-compose-recovery.yaml file is used to recover your node when there is a `server.lock` or `broker.lock` 
+present in the `data` folder after stopping your node.
+
+```sh
+docker compose -f docker-compose-recovery.yaml up --abort-on-container-exit
+```
+
+## Need to resync your node
+
+If recovery failed, get the latest copy for the mainnet node data for linux OS:
+dual: https://catapultmainnetdata.s3.us-west-2.amazonaws.com/weekly/catapult_dual_data.tar.gz
+peer: https://catapultmainnetdata.s3.us-west-2.amazonaws.com/weekly/catapult_peer_data.tar.gz
+
+Note: keep a copy of your node's `harvesters.dat` from the data.

tools/shoestring/shoestring/commands/upgrade.py

@@ -40,6 +40,7 @@ async def run_main(args):
 		harvester_config_patches = _load_harvester_configuration_patches(config_manager)
 
 	(directories.output_directory / 'docker-compose.yaml').unlink()
+	(directories.output_directory / 'docker-compose-recovery.yaml').unlink()
 
 	_purge_directory(directories.userconfig)
 	_recreate_directory(directories.userconfig)

tools/shoestring/shoestring/internal/Preparer.py

@@ -362,6 +362,12 @@ def configure_docker(self, template_mapping):
 		apply_template(compose_template_filename, template_mapping, compose_output_filepath)
 		compose_output_filepath.chmod(0o400)
 
+		recovery_compose_output_filepath = self.directory / 'docker-compose-recovery.yaml'
+		recovery_template_mapping = template_mapping.copy()
+		recovery_template_mapping['recovery'] = True
+		apply_template(compose_template_filename, recovery_template_mapping, recovery_compose_output_filepath)
+		recovery_compose_output_filepath.chmod(0o400)
+
 	def prepare_linking_transaction(self, account_public_key, existing_links, timestamp):
 		"""Creates an aggregate transaction containing account key link and unlink transactions """
 

tools/shoestring/shoestring/startup/startRecovery.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+ulimit -c unlimited
+id -a
+ls -alh /seed
+
+if [ ! -d /seed/00000 ]; then
+	echo "seed directory is empty, exiting"
+	exit 1
+fi
+
+exec /usr/catapult/bin/catapult.recovery /userconfig

tools/shoestring/shoestring/templates/docker-compose-dual.yaml

@@ -33,7 +33,11 @@ services:
       options:
         max-size: 25M
         max-file: '10'
+{%- if recovery %}
+    command: /bin/bash /startup/wait.sh /data/startup/mongo-initialized /startup/startRecovery.sh
+{%- else %}
     command: /bin/bash /startup/wait.sh /data/startup/mongo-initialized /startup/startServer.sh
+{%- endif %}
     environment:
       - LD_LIBRARY_PATH=/usr/catapult/lib:/usr/catapult/deps
     stop_signal: SIGINT
@@ -50,6 +54,7 @@ services:
       - ./logs:/logs
       - ./keys/voting:/votingkeys
 
+{%- if not recovery %}
   broker:
     image: '{{ catapult_client_image }}'
     user: '{{ user }}'
@@ -108,6 +113,7 @@ services:
       - ./https-proxy:/var/lib/https-portal
       - ./https-proxy/nginx.conf.erb:/var/lib/nginx-conf/{{ domainname }}.conf.erb:ro
 {% endif %}
+{% endif %}
 networks:
   default:
     name: catapult-node-network

tools/shoestring/shoestring/templates/docker-compose-peer.yaml

@@ -9,7 +9,11 @@ services:
       options:
         max-size: 25M
         max-file: '10'
+{%- if recovery %}
+    command: /usr/catapult/bin/catapult.recovery /userconfig
+{%- else %}
     command: /usr/catapult/bin/catapult.server /userconfig
+{%- endif %}
     environment:
       - LD_LIBRARY_PATH=/usr/catapult/lib:/usr/catapult/deps
     stop_signal: SIGINT
@@ -26,7 +30,7 @@ services:
       - ./keys/voting:/votingkeys
       - ./keys/cert:/certificates
 
-{%- if light_api %}
+{%- if light_api and not recovery %}
   rest-api:
     image: '{{ catapult_rest_image }}'
     user: '{{ user }}'

tools/shoestring/tests/commands/test_setup.py

@@ -33,6 +33,7 @@ def server(event_loop, aiohttp_client):
 PEER_OUTPUT_FILES = {
 	'data': 0o700,
 	'docker-compose.yaml': 0o400,
+	'docker-compose-recovery.yaml': 0o400,
 	'keys': 0o700,
 	'keys/cert': 0o700,
 	'keys/cert/ca.crt.pem': 0o400,
@@ -89,6 +90,7 @@ def server(event_loop, aiohttp_client):
 	'startup/delayrestapi.sh': 0o400,
 	'startup/mongors.sh': 0o400,
 	'startup/startBroker.sh': 0o400,
+	'startup/startRecovery.sh': 0o400,
 	'startup/startServer.sh': 0o400,
 	'startup/wait.sh': 0o400,
 	'userconfig/resources/config-database.properties': 0o400,

tools/shoestring/tests/commands/test_upgrade.py

@@ -34,6 +34,7 @@ def server(event_loop, aiohttp_client):
 # region changed files
 
 PEER_CHANGED_FILES = [
+	'docker-compose-recovery.yaml',
 	'docker-compose.yaml',
 	'userconfig',
 	'userconfig/resources',
@@ -72,6 +73,7 @@ def server(event_loop, aiohttp_client):
 	'startup/delayrestapi.sh',
 	'startup/mongors.sh',
 	'startup/startBroker.sh',
+	'startup/startRecovery.sh',
 	'startup/startServer.sh',
 	'startup/wait.sh',
 	'userconfig/resources/config-database.properties',

tools/shoestring/tests/internal/test_Preparer.py

@@ -722,7 +722,32 @@ def test_can_generate_random_node_key_when_not_imported(self):
 
 	# region configure_docker
 
-	def _assert_can_configure_docker(self, config, expected_startup_files, expected_service_names):
+	def _assert_can_configure_docker(self, config, expected_startup_files, expected_service_names, expected_recovery_service_names):
+		def _assert_can_configure_service(docker_compose_filepath, expected_names):
+			# - check compose file
+			expected_image_map = {
+				'db': 'mongo:7.0.16',
+				'initiate': 'mongo:7.0.16',
+				'client': 'symbolplatform/symbol-server:gcc-a.b.c.d',
+				'broker': 'symbolplatform/symbol-server:gcc-a.b.c.d',
+				'rest-api': 'symbolplatform/symbol-rest:a.b.c',
+				'rest-api-https-proxy': 'steveltn/https-portal:1'
+			}
+
+			service_names = []
+			with open(docker_compose_filepath, 'rt', encoding='utf8') as infile:
+				compose_config = yaml.safe_load(infile)
+				for service_name in compose_config['services']:
+					service = compose_config['services'][service_name]
+					if 'rest-api-https-proxy' != service_name:
+						self.assertEqual('2222:3333', service['user'])
+
+					self.assertEqual(expected_image_map[service_name], service['image'])
+
+					service_names.append(service_name)
+
+			self.assertEqual(expected_names, service_names)
+
 		# Arrange:
 		with tempfile.TemporaryDirectory() as output_directory:
 			with Preparer(output_directory, config) as preparer:
@@ -747,60 +772,45 @@ def _assert_can_configure_docker(self, config, expected_startup_files, expected_
 				else:
 					self.assertFalse(preparer.directories.startup.exists())
 
-				# - check compose file
-				expected_image_map = {
-					'db': 'mongo:7.0.16',
-					'initiate': 'mongo:7.0.16',
-					'client': 'symbolplatform/symbol-server:gcc-a.b.c.d',
-					'broker': 'symbolplatform/symbol-server:gcc-a.b.c.d',
-					'rest-api': 'symbolplatform/symbol-rest:a.b.c',
-					'rest-api-https-proxy': 'steveltn/https-portal:1'
-				}
-
 				compose_filepath = Path(output_directory) / 'docker-compose.yaml'
 				assert 0o400 == compose_filepath.stat().st_mode & 0o777
+				_assert_can_configure_service(compose_filepath, expected_service_names)
 
-				service_names = []
-				with open(compose_filepath, 'rt', encoding='utf8') as infile:
-					compose_config = yaml.safe_load(infile)
-					for service_name in compose_config['services']:
-						service = compose_config['services'][service_name]
-						if 'rest-api-https-proxy' != service_name:
-							self.assertEqual('2222:3333', service['user'])
-
-						self.assertEqual(expected_image_map[service_name], service['image'])
-
-						service_names.append(service_name)
-
-				self.assertEqual(expected_service_names, service_names)
+				recovery_compose_filepath = Path(output_directory) / 'docker-compose-recovery.yaml'
+				assert 0o400 == recovery_compose_filepath.stat().st_mode & 0o777
+				_assert_can_configure_service(recovery_compose_filepath, expected_recovery_service_names)
 
 	def test_can_configure_docker_peer_node(self):
 		config = self._create_configuration(NodeFeatures.PEER)
-		self._assert_can_configure_docker(config, None, ['client'])
+		self._assert_can_configure_docker(config, None, ['client'], ['client'])
 
 	def test_can_configure_docker_api_node_with_https(self):
 		config = self._create_configuration(NodeFeatures.API)
 		self._assert_can_configure_docker(config, [
-			'delayrestapi.sh', 'mongors.sh', 'startBroker.sh', 'startServer.sh', 'wait.sh'
+			'delayrestapi.sh', 'mongors.sh', 'startBroker.sh', 'startRecovery.sh', 'startServer.sh', 'wait.sh'
 		], [
 			'db', 'initiate', 'client', 'broker', 'rest-api', 'rest-api-https-proxy'
+		], [
+			'db', 'initiate', 'client'
 		])
 
 	def test_can_configure_docker_api_node_without_https(self):
 		config = self._create_configuration(NodeFeatures.API, api_https=False)
 		self._assert_can_configure_docker(config, [
-			'delayrestapi.sh', 'mongors.sh', 'startBroker.sh', 'startServer.sh', 'wait.sh'
+			'delayrestapi.sh', 'mongors.sh', 'startBroker.sh', 'startRecovery.sh', 'startServer.sh', 'wait.sh'
 		], [
 			'db', 'initiate', 'client', 'broker', 'rest-api'
+		], [
+			'db', 'initiate', 'client'
 		])
 
 	def test_can_configure_docker_api_light_node_with_https(self):
 		config = self._create_configuration(NodeFeatures.API, light_api=True)
-		self._assert_can_configure_docker(config, [], ['client', 'rest-api', 'rest-api-https-proxy'])
+		self._assert_can_configure_docker(config, [], ['client', 'rest-api', 'rest-api-https-proxy'], ['client'])
 
 	def test_can_configure_docker_api_light_node_without_https(self):
 		config = self._create_configuration(NodeFeatures.API, api_https=False, light_api=True)
-		self._assert_can_configure_docker(config, [], ['client', 'rest-api'])
+		self._assert_can_configure_docker(config, [], ['client', 'rest-api'], ['client'])
 
 	# endregion