Skip to content

Commit 3da7ea9

Browse files
lizhanhuilizhanhui
authored
[Infra Improvement] Build And Run Unit Tests in Parallel (#4878)
* Support build with Bazel (#4865) * Support build with Bazel, fixing tests to make them capable of running in concurrency and hermetic. * Make test cases capable of running in parallel. (#4874)
1 parent 10d2918 commit 3da7ea9

File tree

110 files changed

+1968
-407
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

110 files changed

+1968
-407
lines changed

‎.bazelrc

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
startup --host_jvm_args=-Xmx2g
18+
19+
run --color=yes
20+
21+
build --color=yes
22+
build --enable_platform_specific_config
23+
24+
test --action_env=TEST_TMPDIR=/tmp
25+
26+
test --experimental_strict_java_deps=warn
27+
build --experimental_strict_java_deps=warn
28+
29+
30+
# This .bazelrc file contains all of the flags required for the provided
31+
# toolchain with Remote Build Execution.
32+
# Note your WORKSPACE must contain an rbe_autoconfig target with
33+
# name="rbe_default" to use these flags as-is.
34+
35+
# Depending on how many machines are in the remote execution instance, setting
36+
# this higher can make builds faster by allowing more jobs to run in parallel.
37+
# Setting it too high can result in jobs that timeout, however, while waiting
38+
# for a remote machine to execute them.
39+
build:remote --jobs=150
40+
41+
# Set several flags related to specifying the platform, toolchain and java
42+
# properties.
43+
# These flags should only be used as is for the rbe-ubuntu16-04 container
44+
# and need to be adapted to work with other toolchain containers.
45+
build:remote --java_runtime_version=rbe_jdk
46+
build:remote --tool_java_runtime_version=rbe_jdk
47+
build:remote --extra_toolchains=@rbe_default//java:all
48+
49+
build:remote --crosstool_top=@rbe_default//cc:toolchain
50+
build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
51+
# Platform flags:
52+
# The toolchain container used for execution is defined in the target indicated
53+
# by "extra_execution_platforms", "host_platform" and "platforms".
54+
# More about platforms: https://docs.bazel.build/versions/master/platforms.html
55+
build:remote --extra_toolchains=@rbe_default//config:cc-toolchain
56+
build:remote --extra_execution_platforms=@rbe_default//config:platform
57+
build:remote --host_platform=@rbe_default//config:platform
58+
build:remote --platforms=@rbe_default//config:platform
59+
60+
# Starting with Bazel 0.27.0 strategies do not need to be explicitly
61+
# defined. See https://github.com/bazelbuild/bazel/issues/7480
62+
build:remote --define=EXECUTOR=remote
63+
64+
# Enable remote execution so actions are performed on the remote systems.
65+
build:remote --remote_executor=grpcs://remote.buildbuddy.io
66+
67+
# Enforce stricter environment rules, which eliminates some non-hermetic
68+
# behavior and therefore improves both the remote cache hit rate and the
69+
# correctness and repeatability of the build.
70+
build:remote --incompatible_strict_action_env=true
71+
72+
# Set a higher timeout value, just in case.
73+
build:remote --remote_timeout=3600

‎.bazelversion

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5.2.0

‎.github/workflows/bazel.yml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: Build and Run Tests By Bazel
2+
on:
3+
pull_request_target:
4+
types: [opened, reopened, synchronize]
5+
push:
6+
branches:
7+
- master
8+
- develop
9+
- bazel
10+
jobs:
11+
build:
12+
name: "Java (${{ matrix.os }})"
13+
runs-on: ${{ matrix.os }}
14+
strategy:
15+
matrix:
16+
os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04]
17+
steps:
18+
- uses: actions/checkout@v2
19+
- name: Build
20+
run: bazel build --config=remote --remote_header=x-buildbuddy-api-key=${{ secrets.BUILD_BUDDY_API_KEY }} //...
21+
- name: Run Tests
22+
run: bazel test --config=remote --remote_header=x-buildbuddy-api-key=${{ secrets.BUILD_BUDDY_API_KEY }} //...

‎.github/workflows/build.yaml ‎.github/workflows/maven.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
name: Build
1+
name: Build and Run Tests By Maven
22
on:
33
pull_request:
44
types: [opened, reopened, synchronize]
55
push:
6-
branches: [master, develop]
6+
branches: [master, develop, bazel]
77
jobs:
88
java_build:
99
name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})"

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,8 @@ devenv
1313
.DS_Store
1414
localbin
1515
nohup.out
16+
bazel-out
17+
bazel-bin
18+
bazel-rocketmq
19+
bazel-testlogs
20+
.vscode

‎.licenserc.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ header:
4242
- 'distribution/LICENSE-BIN'
4343
- 'distribution/NOTICE-BIN'
4444
- 'distribution/conf/rmq-proxy.json'
45+
- '.bazelversion'
4546

4647

4748
comment: on-failure

‎BUILD.bazel

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_rbe_exec_properties_dict")
18+
19+
platform(
20+
name = "custom_platform",
21+
# Inherit from the platform target generated by 'rbe_configs_gen' assuming the generated configs
22+
# were imported as a Bazel external repository named 'rbe_default'. If you extracted the
23+
# generated configs elsewhere in your source repository, replace the following with the label
24+
# to the 'platform' target in the generated configs.
25+
parents = ["@rbe_default//config:platform"],
26+
# Example custom execution property instructing RBE to use e2-standard-2 GCE VMs.
27+
exec_properties = create_rbe_exec_properties_dict(
28+
container_image = "ubuntu:latest",
29+
),
30+
)
31+
32+
java_library(
33+
name = "test_deps",
34+
visibility = ["//visibility:public"],
35+
exports = [
36+
"@maven//:junit_junit",
37+
"@maven//:org_assertj_assertj_core",
38+
"@maven//:org_hamcrest_hamcrest_library",
39+
"@maven//:org_mockito_mockito_core",
40+
"@maven//:org_hamcrest_hamcrest_core",
41+
"@maven//:ch_qos_logback_logback_classic",
42+
"@maven//:org_awaitility_awaitility",
43+
"@maven//:org_openjdk_jmh_jmh_core",
44+
"@maven//:org_openjdk_jmh_jmh_generator_annprocess",
45+
],
46+
)

‎WORKSPACE

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
18+
19+
RULES_JVM_EXTERNAL_TAG = "4.2"
20+
21+
RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca"
22+
23+
http_archive(
24+
name = "rules_jvm_external",
25+
sha256 = RULES_JVM_EXTERNAL_SHA,
26+
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
27+
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
28+
)
29+
30+
load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
31+
32+
rules_jvm_external_deps()
33+
34+
load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
35+
36+
rules_jvm_external_setup()
37+
38+
load("@rules_jvm_external//:defs.bzl", "maven_install")
39+
40+
maven_install(
41+
artifacts = [
42+
"junit:junit:4.13.2",
43+
"com.alibaba:fastjson:1.2.76",
44+
"org.hamcrest:hamcrest-library:1.3",
45+
"io.netty:netty-all:4.1.65.Final",
46+
"org.slf4j:slf4j-api:1.7.7",
47+
"org.assertj:assertj-core:3.22.0",
48+
"org.mockito:mockito-core:3.10.0",
49+
"com.github.luben:zstd-jni:1.5.2-2",
50+
"org.lz4:lz4-java:1.8.0",
51+
"commons-validator:commons-validator:1.7",
52+
"org.apache.commons:commons-lang3:3.4",
53+
"org.hamcrest:hamcrest-core:1.3",
54+
# "io.openmessaging.storage:dledger:0.2.4",
55+
"net.java.dev.jna:jna:4.2.2",
56+
"ch.qos.logback:logback-classic:1.2.10",
57+
"ch.qos.logback:logback-core:1.2.10",
58+
"io.opentracing:opentracing-api:0.33.0",
59+
"io.opentracing:opentracing-mock:0.33.0",
60+
"commons-collections:commons-collections:3.2.2",
61+
"org.awaitility:awaitility:4.1.0",
62+
"commons-cli:commons-cli:1.4",
63+
"com.google.guava:guava:31.0.1-jre",
64+
"org.yaml:snakeyaml:1.30",
65+
"commons-codec:commons-codec:1.13",
66+
"commons-io:commons-io:2.7",
67+
"log4j:log4j:1.2.17",
68+
"com.google.truth:truth:0.30",
69+
"org.bouncycastle:bcpkix-jdk15on:1.69",
70+
"com.google.code.gson:gson:2.8.9",
71+
"com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.4.2",
72+
"org.apache.rocketmq:rocketmq-proto:2.0.0",
73+
"com.google.protobuf:protobuf-java:3.20.1",
74+
"com.google.protobuf:protobuf-java-util:3.20.1",
75+
"com.conversantmedia:disruptor:1.2.10",
76+
"javax.annotation:javax.annotation-api:1.3.2",
77+
"com.google.code.findbugs:jsr305:3.0.2",
78+
"org.checkerframework:checker-qual:3.12.0",
79+
"org.reflections:reflections:0.9.11",
80+
"org.openjdk.jmh:jmh-core:1.19",
81+
"org.openjdk.jmh:jmh-generator-annprocess:1.19",
82+
"com.github.ben-manes.caffeine:caffeine:2.9.3",
83+
"io.grpc:grpc-services:1.47.0",
84+
"io.grpc:grpc-netty-shaded:1.47.0",
85+
"io.grpc:grpc-context:1.47.0",
86+
"io.grpc:grpc-stub:1.47.0",
87+
"io.grpc:grpc-api:1.47.0",
88+
"io.grpc:grpc-testing:1.47.0",
89+
],
90+
fetch_sources = True,
91+
repositories = [
92+
# Private repositories are supported through HTTP Basic auth
93+
"https://repo1.maven.org/maven2",
94+
],
95+
)
96+
97+
http_archive(
98+
name = "io_buildbuddy_buildbuddy_toolchain",
99+
sha256 = "a2a5cccec251211e2221b1587af2ce43c36d32a42f5d881737db3b546a536510",
100+
strip_prefix = "buildbuddy-toolchain-829c8a574f706de5c96c54ca310f139f4acda7dd",
101+
urls = ["https://github.com/buildbuddy-io/buildbuddy-toolchain/archive/829c8a574f706de5c96c54ca310f139f4acda7dd.tar.gz"],
102+
)
103+
104+
load("@io_buildbuddy_buildbuddy_toolchain//:deps.bzl", "buildbuddy_deps")
105+
106+
buildbuddy_deps()
107+
108+
load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy")
109+
110+
buildbuddy(name = "buildbuddy_toolchain")
111+
112+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
113+
114+
http_archive(
115+
name = "rbe_default",
116+
# sha256 = "c0d428774cbe70d477e1d07581d863f8dbff4ba6a66d20502d7118354a814bea",
117+
urls = ["https://storage.googleapis.com/rbe-toolchain/bazel-configs/rbe-ubuntu1604/latest/rbe_default.tar"],
118+
)
119+
120+
http_archive(
121+
name = "bazel_toolchains",
122+
urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/dac71231098d891e5c4b74a2078fe9343feef510.tar.gz"],
123+
strip_prefix = "bazel-toolchains-dac71231098d891e5c4b74a2078fe9343feef510",
124+
sha256 = "56d5370eb99559b4c74f334f81bc8a298f728bd16d5a4333c865c2ad10fae3bc",
125+
)
126+
127+
load("@bazel_toolchains//repositories:repositories.bzl", bazel_toolchains_repositories = "repositories")
128+
bazel_toolchains_repositories()

‎acl/BUILD.bazel

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
load("//bazel:GenTestRules.bzl", "GenTestRules")
18+
19+
java_library(
20+
name = "acl",
21+
srcs = glob(["src/main/java/**/*.java"]),
22+
visibility = ["//visibility:public"],
23+
deps = [
24+
"//common",
25+
"//remoting",
26+
"//logging",
27+
"//srvutil",
28+
"@maven//:org_apache_commons_commons_lang3",
29+
"@maven//:commons_validator_commons_validator",
30+
"@maven//:com_github_luben_zstd_jni",
31+
"@maven//:org_lz4_lz4_java",
32+
"@maven//:com_alibaba_fastjson",
33+
"@maven//:io_netty_netty_all",
34+
"@maven//:org_yaml_snakeyaml",
35+
"@maven//:commons_codec_commons_codec",
36+
"@maven//:org_apache_rocketmq_rocketmq_proto",
37+
"@maven//:com_google_protobuf_protobuf_java",
38+
"@maven//:com_google_guava_guava",
39+
],
40+
)
41+
42+
java_library(
43+
name = "tests",
44+
srcs = glob(["src/test/java/**/*.java"]),
45+
visibility = ["//visibility:public"],
46+
deps = [
47+
":acl",
48+
"//common",
49+
"//remoting",
50+
"//:test_deps",
51+
"@maven//:org_apache_commons_commons_lang3",
52+
"@maven//:io_netty_netty_all",
53+
"@maven//:org_yaml_snakeyaml",
54+
"@maven//:commons_codec_commons_codec",
55+
"@maven//:com_alibaba_fastjson",
56+
],
57+
resources = glob(["src/test/resources/**/*.yml"])
58+
)
59+
60+
GenTestRules(
61+
name = "GeneratedTestRules",
62+
test_files = glob(["src/test/java/**/*Test.java"]),
63+
deps = [
64+
":tests",
65+
],
66+
# The following tests are not hermetic. Fix them later.
67+
exclude_tests = [
68+
"src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest",
69+
"src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest",
70+
"src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest",
71+
],
72+
)

0 commit comments

Comments
 (0)