Skip to content

Commit 7976eb0

Browse files
authored
[tools/shoestring]: light rest implementation
Problem: The light REST functionality in the tools is missing. solution: Added light rest configuration in the tools
1 parent 3142777 commit 7976eb0

21 files changed

+236
-43
lines changed

tools/shoestring/shoestring/commands/reset_data.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33

44
from zenlog import log
55

6-
from shoestring.internal.NodeFeatures import NodeFeatures
76
from shoestring.internal.ShoestringConfiguration import parse_shoestring_configuration
87

98

@@ -77,7 +76,7 @@ async def run_main(args):
7776
for name in ('data', 'logs'):
7877
_purge_and_recreate(directory / name)
7978

80-
if NodeFeatures.API in config.node.features:
79+
if config.node.full_api:
8180
_purge_and_recreate(directory / 'dbdata')
8281

8382
stateful_data_processor.restore()

tools/shoestring/shoestring/commands/setup.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ async def run_main(args):
9292
api_endpoints = await download_peers(
9393
config.services.nodewatch,
9494
preparer.directories.resources,
95-
NodeFeatures.API in config.node.features)
95+
config.node.full_api)
9696
await download_and_extract_package(args.package, preparer.directories.temp)
9797

9898
# prepare nemesis data and resources
@@ -121,6 +121,7 @@ async def run_main(args):
121121
'catapult_rest_image': config.images.rest,
122122
'user': f'{config.node.user_id}:{config.node.group_id}',
123123
'api_https': config.node.api_https,
124+
'light_api': NodeFeatures.API in config.node.features and not config.node.full_api,
124125
'domainname': hostname
125126
})
126127

tools/shoestring/shoestring/commands/upgrade.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,12 @@ async def run_main(args):
4545
_recreate_directory(directories.userconfig)
4646
_recreate_directory(directories.resources)
4747

48-
if NodeFeatures.API in config.node.features:
48+
if config.node.full_api:
4949
_purge_directory(directories.startup)
5050
_purge_directory(directories.mongo)
5151

52-
if config.node.api_https:
53-
(directories.https_proxy / 'nginx.conf.erb').unlink()
52+
if NodeFeatures.API in config.node.features and config.node.api_https:
53+
(directories.https_proxy / 'nginx.conf.erb').unlink()
5454

5555
await run_setup_main(args)
5656

tools/shoestring/shoestring/healthagents/websockets.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,12 @@
55
from zenlog import log
66

77
from shoestring.internal.ConfigurationManager import parse_time_span
8-
from shoestring.internal.NodeFeatures import NodeFeatures
98

109
NAME = 'REST websockets'
1110

1211

1312
def should_run(node_config):
14-
return NodeFeatures.API in node_config.features
13+
return node_config.full_api
1514

1615

1716
async def validate(context):

tools/shoestring/shoestring/internal/Preparer.py

+22-9
Original file line numberDiff line numberDiff line change
@@ -164,12 +164,15 @@ def create_subdirectories(self):
164164
self.directories.userconfig,
165165
self.directories.resources
166166
]
167+
167168
if NodeFeatures.API in self.config.node.features:
168-
directories.append(self.directories.dbdata)
169169
directories.append(self.directories.rest_cache)
170170
if self.config.node.api_https:
171171
directories.append(self.directories.https_proxy)
172172

173+
if self.config.node.full_api:
174+
directories.append(self.directories.dbdata)
175+
173176
if NodeFeatures.VOTER in self.config.node.features:
174177
directories.append(self.directories.voting_keys)
175178

@@ -217,7 +220,7 @@ def prepare_resources(self):
217220

218221
self._copy_properties_files(PEER_EXTENSIONS)
219222

220-
if NodeFeatures.API in self.config.node.features:
223+
if self.config.node.full_api:
221224
self._copy_properties_files(API_EXTENSIONS)
222225

223226
if NodeFeatures.HARVESTER in self.config.node.features:
@@ -231,6 +234,14 @@ def configure_resources(self, user_patches=None):
231234
"""Configures resources based on enabled features."""
232235

233236
if NodeFeatures.API in self.config.node.features:
237+
self._patch_resources({
238+
'node': [
239+
('node', 'trustedHosts', '127.0.0.1,172.20.0.25'),
240+
('node', 'localNetworks', '127.0.0.1,172.20'),
241+
]
242+
})
243+
244+
if self.config.node.full_api:
234245
self._patch_resources({
235246
'extensions-server': [
236247
('extensions', 'extension.filespooling', 'true'),
@@ -243,8 +254,6 @@ def configure_resources(self, user_patches=None):
243254
],
244255
'node': [
245256
('node', 'enableAutoSyncCleanup', 'false'),
246-
('node', 'trustedHosts', '127.0.0.1,172.20.0.25'),
247-
('node', 'localNetworks', '127.0.0.1,172.20'),
248257
('localnode', 'roles', 'Peer,Api'),
249258
]
250259
})
@@ -273,10 +282,12 @@ def configure_rest(self, rest_overrides_filename=None):
273282
if NodeFeatures.API not in self.config.node.features:
274283
return
275284

276-
self._copy_tree_readonly(self.directories.temp / 'mongo', self.directories.mongo)
277-
self._make_files_readonly(self.directories.mongo)
285+
rest_file_name = 'rest' if self.config.node.full_api else 'rest-light'
286+
self._copy_file(self.directories.temp / 'rest' / f'{rest_file_name}.json', self.directories.userconfig / 'rest.json')
278287

279-
self._copy_file(self.directories.temp / 'rest' / 'rest.json', self.directories.userconfig)
288+
if self.config.node.full_api:
289+
self._copy_tree_readonly(self.directories.temp / 'mongo', self.directories.mongo)
290+
self._make_files_readonly(self.directories.mongo)
280291

281292
if rest_overrides_filename:
282293
rest_json_filepath = self.directories.userconfig / 'rest.json'
@@ -340,10 +351,12 @@ def generate_certificates(self, ca_key_path, require_ca=True):
340351
def configure_docker(self, template_mapping):
341352
"""Prepares docker-compose file."""
342353

343-
if NodeFeatures.API in self.config.node.features:
354+
if self.config.node.full_api:
344355
self._copy_tree_readonly(_qualify_resource('startup'), self.directories.startup)
356+
compose_template_filename_postfix = 'dual'
357+
else:
358+
compose_template_filename_postfix = 'peer'
345359

346-
compose_template_filename_postfix = 'dual' if NodeFeatures.API in self.config.node.features else 'peer'
347360
compose_template_filename = _qualify_resource(f'templates/docker-compose-{compose_template_filename_postfix}.yaml')
348361
compose_output_filepath = self.directory / 'docker-compose.yaml'
349362
apply_template(compose_template_filename, template_mapping, compose_output_filepath)

tools/shoestring/shoestring/internal/ShoestringConfiguration.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
])
1515
ImportsConfiguration = namedtuple('ImportsConfiguration', ['harvester', 'voter', 'node_key'])
1616
NodeConfiguration = namedtuple('NodeConfiguration', [
17-
'features', 'user_id', 'group_id', 'ca_password', 'api_https', 'ca_common_name', 'node_common_name'
17+
'features', 'user_id', 'group_id', 'ca_password', 'api_https', 'full_api', 'ca_common_name', 'node_common_name'
1818
])
1919
ShoestringConfiguration = namedtuple('ShoestringConfiguration', ['network', 'images', 'services', 'transaction', 'imports', 'node'])
2020

@@ -74,10 +74,11 @@ def parse_node_configuration(config):
7474
group_id = int(config['groupId'])
7575
ca_password = config['caPassword']
7676
api_https = config['apiHttps'].lower() == 'true'
77+
full_api = NodeFeatures.API in features and not config['lightApi'].lower() == 'true'
7778
ca_common_name = config['caCommonName']
7879
node_common_name = config['nodeCommonName']
7980

80-
return NodeConfiguration(features, user_id, group_id, ca_password, api_https, ca_common_name, node_common_name)
81+
return NodeConfiguration(features, user_id, group_id, ca_password, api_https, full_api, ca_common_name, node_common_name)
8182

8283

8384
def parse_shoestring_configuration(filename):

tools/shoestring/shoestring/templates/docker-compose-peer.yaml

+49
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,52 @@ services:
2525
- ./logs:/logs
2626
- ./keys/voting:/votingkeys
2727
- ./keys/cert:/certificates
28+
29+
{%- if light_api %}
30+
rest-api:
31+
image: '{{ catapult_rest_image }}'
32+
user: '{{ user }}'
33+
command: npm run start-light --prefix /app /userconfig/rest.json
34+
environment:
35+
- HOME=/symbol-workdir
36+
- NODE_ENV=production
37+
working_dir: /symbol-workdir
38+
{%- if not api_https %}
39+
ports:
40+
- 3000:3000
41+
{%- endif %}
42+
volumes:
43+
- ./startup:/startup:ro
44+
- ./userconfig:/userconfig:ro
45+
- ./keys/cert:/certificates:ro
46+
- ./data:/data
47+
- ./logs:/logs
48+
- ./rest-cache:/symbol-workdir
49+
networks:
50+
default:
51+
ipv4_address: 172.20.0.25
52+
{%- if api_https %}
53+
rest-api-https-proxy:
54+
image: steveltn/https-portal:1
55+
ports:
56+
- 80:80
57+
- 3000:80
58+
- 3001:443
59+
links:
60+
- rest-api
61+
environment:
62+
- DOMAINS={{ domainname }} -> http://rest-api:3000
63+
- WEBSOCKET=false
64+
- STAGE=production
65+
- FORCE_RENEW=false
66+
volumes:
67+
- ./https-proxy:/var/lib/https-portal
68+
- ./https-proxy/nginx.conf.erb:/var/lib/nginx-conf/{{ domainname }}.conf.erb:ro
69+
{%- endif %}
70+
networks:
71+
default:
72+
name: catapult-node-network
73+
ipam:
74+
config:
75+
- subnet: 172.20.0.0/24
76+
{%- endif %}

tools/shoestring/tests/commands/test_health.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def _create_configuration(api_https):
3333
return ShoestringConfiguration(
3434
*(4 * [None]),
3535
ImportsConfiguration(None, None, None),
36-
NodeConfiguration(NodeFeatures.PEER, None, None, None, api_https, 'CA', 'NODE'))
36+
NodeConfiguration(NodeFeatures.PEER, None, None, None, api_https, False, 'CA', 'NODE'))
3737

3838

3939
# pylint: disable=invalid-name

tools/shoestring/tests/commands/test_reset_data.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@ def _create_directories_with_placeholders(directory, subdirectory_names):
2121

2222
# region basic
2323

24-
async def _assert_reset_data(node_features, expected_recreated_subdirectories):
24+
async def _assert_reset_data(node_features, expected_recreated_subdirectories, light_api=False):
2525
# Arrange:
2626
subdirectory_names = ('data', 'logs', 'dbdata', 'keys', 'unknown')
2727
with tempfile.TemporaryDirectory() as output_directory:
28-
config_filepath = prepare_shoestring_configuration(output_directory, node_features)
28+
config_filepath = prepare_shoestring_configuration(output_directory, node_features, light_api=light_api)
2929

3030
# - create some directories each with a placeholder file
3131
_create_directories_with_placeholders(output_directory, subdirectory_names)
@@ -59,6 +59,10 @@ async def test_can_reset_data_api_node():
5959
await _assert_reset_data(NodeFeatures.API, ['data', 'logs', 'dbdata'])
6060

6161

62+
async def test_can_reset_data_light_api_node():
63+
await _assert_reset_data(NodeFeatures.API, ['data', 'logs'], light_api=True)
64+
65+
6266
async def test_can_reset_data_voter_node_without_voter_state():
6367
await _assert_reset_data(NodeFeatures.VOTER, ['data', 'logs'])
6468

tools/shoestring/tests/commands/test_setup.py

+5
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,11 @@ def server(event_loop, aiohttp_client):
100100
'userconfig/rest.json': 0o400
101101
}
102102

103+
LIGHT_API_OUTPUT_FILES = {
104+
'userconfig/rest.json': 0o400,
105+
'rest-cache': 0o700
106+
}
107+
103108
HARVESTER_OUTPUT_FILES = {
104109
'keys/remote.pem': 0o400,
105110
'keys/vrf.pem': 0o400,

tools/shoestring/tests/commands/test_upgrade.py

+21-3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
API_OUTPUT_FILES,
1616
HARVESTER_OUTPUT_FILES,
1717
HTTPS_OUTPUT_FILES,
18+
LIGHT_API_OUTPUT_FILES,
1819
PEER_OUTPUT_FILES,
1920
STATE_CHANGE_OUTPUT_FILES,
2021
VOTER_OUTPUT_FILES
@@ -82,6 +83,10 @@ def server(event_loop, aiohttp_client):
8283
'userconfig/rest.json'
8384
]
8485

86+
LIGHT_API_CHANGED_FILES = [
87+
'userconfig/rest.json'
88+
]
89+
8590
HARVESTER_CHANGED_FILES = [
8691
'userconfig/resources/config-harvesting.properties'
8792
]
@@ -135,12 +140,13 @@ async def _assert_can_upgrade_node(
135140
node_features,
136141
expected_output_files,
137142
expected_changed_files,
138-
api_https=False
139-
):
143+
api_https=False,
144+
light_api=False
145+
): # pylint: disable=too-many-arguments,too-many-positional-arguments
140146
# Arrange:
141147
with tempfile.TemporaryDirectory() as output_directory:
142148
with tempfile.TemporaryDirectory() as package_directory:
143-
prepare_shoestring_configuration(package_directory, node_features, server.make_url(''), api_https=api_https)
149+
prepare_shoestring_configuration(package_directory, node_features, server.make_url(''), api_https=api_https, light_api=light_api)
144150
_prepare_overrides(package_directory, 'name from setup')
145151
prepare_testnet_package(package_directory, 'resources.zip')
146152

@@ -210,6 +216,18 @@ async def test_can_upgrade_api_node_with_https(server): # pylint: disable=redef
210216
await _assert_can_upgrade_node(server, NodeFeatures.API, expected_output_files, expected_changed_files, api_https=True)
211217

212218

219+
async def test_can_upgrade_light_api_node(server): # pylint: disable=redefined-outer-name
220+
expected_output_files = {**PEER_OUTPUT_FILES, **LIGHT_API_OUTPUT_FILES}
221+
expected_changed_files = sorted(PEER_CHANGED_FILES + LIGHT_API_CHANGED_FILES)
222+
await _assert_can_upgrade_node(server, NodeFeatures.API, expected_output_files, expected_changed_files, light_api=True)
223+
224+
225+
async def test_can_upgrade_light_api_node_with_https(server): # pylint: disable=redefined-outer-name
226+
expected_output_files = {**PEER_OUTPUT_FILES, **LIGHT_API_OUTPUT_FILES, **HTTPS_OUTPUT_FILES}
227+
expected_changed_files = sorted(PEER_CHANGED_FILES + LIGHT_API_CHANGED_FILES + HTTPS_CHANGED_FILES)
228+
await _assert_can_upgrade_node(server, NodeFeatures.API, expected_output_files, expected_changed_files, api_https=True, light_api=True)
229+
230+
213231
async def test_can_upgrade_harvester_node(server): # pylint: disable=redefined-outer-name
214232
expected_output_files = {**PEER_OUTPUT_FILES, **HARVESTER_OUTPUT_FILES, **STATE_CHANGE_OUTPUT_FILES}
215233
expected_changed_files = sorted(PEER_CHANGED_FILES + HARVESTER_CHANGED_FILES)

tools/shoestring/tests/healthagents/test_peer_api.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020

2121
def _create_configuration():
22-
node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, 'CA CN', 'NODE CN')
22+
node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, None, 'CA CN', 'NODE CN')
2323
return ShoestringConfiguration('testnet', *(3 * [None]), ImportsConfiguration(None, None, None), node_config)
2424

2525

@@ -83,7 +83,7 @@ async def handle_packet(reader, writer):
8383
def test_should_run_for_all_roles():
8484
# Act + Assert:
8585
for features in (NodeFeatures.PEER, NodeFeatures.API, NodeFeatures.HARVESTER, NodeFeatures.VOTER):
86-
assert should_run(NodeConfiguration(features, *([None] * 6))), str(features)
86+
assert should_run(NodeConfiguration(features, *([None] * 7))), str(features)
8787

8888
# endregion
8989

tools/shoestring/tests/healthagents/test_peer_certificate.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
def test_should_run_for_all_roles():
2323
# Act + Assert:
2424
for features in (NodeFeatures.PEER, NodeFeatures.API, NodeFeatures.HARVESTER, NodeFeatures.VOTER):
25-
assert should_run(NodeConfiguration(features, *([None] * 6))), str(features)
25+
assert should_run(NodeConfiguration(features, *([None] * 7))), str(features)
2626

2727
# endregion
2828

@@ -34,7 +34,7 @@ def _create_executor():
3434

3535

3636
def _create_configuration():
37-
node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, 'CA CN', 'NODE CN')
37+
node_config = NodeConfiguration(NodeFeatures.PEER, None, None, None, None, None, 'CA CN', 'NODE CN')
3838
return ShoestringConfiguration('testnet', *(3 * [None]), ImportsConfiguration(None, None, None), node_config)
3939

4040

tools/shoestring/tests/healthagents/test_rest_api.py

+8-2
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,16 @@ async def _process(_, response_body):
5555

5656
def test_should_run_for_api_role():
5757
# Act + Assert:
58-
assert should_run(NodeConfiguration(NodeFeatures.API, *([None] * 6)))
58+
assert should_run(NodeConfiguration(NodeFeatures.API, None, None, None, None, True, None, None))
5959

6060
for features in (NodeFeatures.PEER, NodeFeatures.HARVESTER, NodeFeatures.VOTER):
61-
assert not should_run(NodeConfiguration(features, *([None] * 6))), str(features)
61+
assert not should_run(NodeConfiguration(features, *([None] * 7))), str(features)
62+
63+
64+
def test_should_run_for_light_node():
65+
# Act + Assert:
66+
assert should_run(NodeConfiguration(NodeFeatures.API, None, None, None, None, False, None, None))
67+
6268

6369
# endregion
6470

tools/shoestring/tests/healthagents/test_rest_https_certificate.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ async def server(aiohttp_server):
3434
# region should_run
3535

3636
def _create_node_configuration(api_https):
37-
return NodeConfiguration(None, None, None, None, api_https, None, None)
37+
return NodeConfiguration(None, None, None, None, api_https, None, None, None)
3838

3939

4040
def test_should_run_for_https_role():

tools/shoestring/tests/healthagents/test_voting_keys.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ def server(event_loop, aiohttp_client):
3333

3434
def test_should_run_for_voter_role():
3535
# Act + Assert:
36-
assert should_run(NodeConfiguration(NodeFeatures.VOTER, *([None] * 6)))
36+
assert should_run(NodeConfiguration(NodeFeatures.VOTER, *([None] * 7)))
3737

3838
for features in (NodeFeatures.PEER, NodeFeatures.API, NodeFeatures.HARVESTER):
39-
assert not should_run(NodeConfiguration(features, *([None] * 6))), str(features)
39+
assert not should_run(NodeConfiguration(features, *([None] * 7))), str(features)
4040

4141
# endregion
4242

0 commit comments

Comments
 (0)