Skip to content

Commit 1fc74a2

Browse files
author
redyyang
committed
fix(wasm): align bridge with merged lookup policy and verify fixtures.
2 parents 0fd7965 + a89459b commit 1fc74a2

24 files changed

Lines changed: 561 additions & 81 deletions

File tree

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
"""Rebuild WAT fixtures exactly and test freshly compiled legacy Rust guests."""
17+
18+
import argparse
19+
import shutil
20+
import subprocess
21+
import tempfile
22+
from pathlib import Path
23+
24+
ROOT = Path(__file__).resolve().parents[2]
25+
PLUGIN = ROOT / "shenyu-plugin"
26+
API = PLUGIN / "shenyu-plugin-wasm-api" / "src/test"
27+
BASE = PLUGIN / "shenyu-plugin-wasm-base" / "src/test"
28+
RUST_TOOLCHAIN = "1.90.0"
29+
RUST_GUESTS = [
30+
(API, "rust-wasm-plugin", "org.apache.shenyu.plugin.wasm.api.AbstractWasmPluginTest$RustWasmPlugin.wasm"),
31+
(BASE, "rust-shenyu-wasm-plugin", "org.apache.shenyu.plugin.wasm.base.AbstractShenyuWasmPluginTest$TestShenyuWasmPlugin.wasm"),
32+
(BASE, "rust-plugin-data-handler", "org.apache.shenyu.plugin.wasm.base.handler.AbstractWasmPluginDataHandlerTest$TestWasmPluginDataHandler.wasm"),
33+
(BASE, "rust-meta-data-handler", "org.apache.shenyu.plugin.wasm.base.handler.AbstractWasmMetaDataHandlerTest$TestWasmMetaDataHandler.wasm"),
34+
(BASE, "rust-discovery-handler", "org.apache.shenyu.plugin.wasm.base.handler.AbstractWasmDiscoveryHandlerTest$TestWasmPluginDiscoveryHandler.wasm"),
35+
]
36+
37+
38+
def run(command, **kwargs):
39+
subprocess.run(command, check=True, **kwargs)
40+
41+
42+
def verify_wat():
43+
version = subprocess.check_output(["wat2wasm", "--version"], text=True).strip()
44+
if version != "1.0.42":
45+
raise RuntimeError("WAT verification requires WABT 1.0.42, found " + version)
46+
count = 0
47+
for family in ["wasm-bridge", "wasm-gateway"]:
48+
directory = BASE / family
49+
if not directory.exists():
50+
continue
51+
with tempfile.TemporaryDirectory(prefix="shenyu-wat-") as temporary:
52+
run(["make", "-C", str(directory), "build", "RESOURCES=" + temporary])
53+
binaries = list(Path(temporary).glob("*.wasm"))
54+
if not binaries:
55+
raise RuntimeError("No fixtures generated for " + family)
56+
for rebuilt in binaries:
57+
committed = BASE / "resources" / rebuilt.name
58+
if rebuilt.read_bytes() != committed.read_bytes():
59+
raise RuntimeError("WAT fixture differs from committed binary: " + rebuilt.name)
60+
count += 1
61+
if not count:
62+
raise RuntimeError("No WAT fixture families found")
63+
print("Verified " + str(count) + " byte-identical WAT fixtures", flush=True)
64+
65+
66+
def verify_rust():
67+
originals = {}
68+
try:
69+
with tempfile.TemporaryDirectory(prefix="shenyu-rust-") as temporary:
70+
for source_root, crate, resource_name in RUST_GUESTS:
71+
source = source_root / crate
72+
copied = Path(temporary) / crate
73+
copied.mkdir()
74+
shutil.copyfile(source / "Cargo.toml", copied / "Cargo.toml")
75+
shutil.copytree(source / "src", copied / "src")
76+
run(["cargo", "+" + RUST_TOOLCHAIN, "build", "--manifest-path", str(copied / "Cargo.toml"),
77+
"--target", "wasm32-wasip1", "--release", "--offline"])
78+
binary = copied / "target/wasm32-wasip1/release" / (crate.replace("-", "_") + ".wasm")
79+
resource = source_root / "resources" / resource_name
80+
originals[resource] = resource.read_bytes()
81+
shutil.copyfile(binary, resource)
82+
# Maven copies the freshly rebuilt guests before the tests load them.
83+
# Legacy compiler versions were not recorded, so assert behavior instead of bytes.
84+
run(["./mvnw", "-B", "-ntp", "-pl", "shenyu-plugin/shenyu-plugin-wasm-base", "-am", "test",
85+
"-Dtest=*Wasm*Test", "-Dsurefire.failIfNoSpecifiedTests=false",
86+
"-DskipRemoteResources=true", "-Djacoco.skip=true", "-Dmaven.javadoc.skip=true"], cwd=ROOT)
87+
finally:
88+
for resource, original in originals.items():
89+
resource.write_bytes(original)
90+
print("Verified five rebuilt Rust guests through the WASM Java suites", flush=True)
91+
92+
93+
def main():
94+
parser = argparse.ArgumentParser(description=__doc__)
95+
mode = parser.add_mutually_exclusive_group()
96+
mode.add_argument("--wat-only", action="store_true")
97+
mode.add_argument("--rust-only", action="store_true")
98+
options = parser.parse_args()
99+
if not options.rust_only:
100+
verify_wat()
101+
if not options.wat_only:
102+
verify_rust()
103+
104+
105+
if __name__ == "__main__":
106+
main()
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: wasm-fixtures
17+
18+
on:
19+
pull_request:
20+
paths:
21+
- 'shenyu-plugin/shenyu-plugin-wasm-api/**'
22+
- 'shenyu-plugin/shenyu-plugin-wasm-base/**'
23+
- '.github/scripts/verify-wasm-fixtures.py'
24+
- '.github/workflows/wasm-fixtures.yml'
25+
- 'pom.xml'
26+
push:
27+
branches: [master]
28+
paths:
29+
- 'shenyu-plugin/shenyu-plugin-wasm-api/**'
30+
- 'shenyu-plugin/shenyu-plugin-wasm-base/**'
31+
- '.github/scripts/verify-wasm-fixtures.py'
32+
- '.github/workflows/wasm-fixtures.yml'
33+
- 'pom.xml'
34+
35+
permissions:
36+
contents: read
37+
38+
concurrency:
39+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
40+
cancel-in-progress: true
41+
42+
jobs:
43+
rebuild-and-test:
44+
runs-on: ubuntu-latest
45+
timeout-minutes: 30
46+
steps:
47+
- uses: actions/checkout@v3
48+
- uses: ./actions/setup-java-with-retry
49+
with:
50+
java-version: "17"
51+
distribution: "temurin"
52+
- name: Install pinned WABT
53+
shell: bash
54+
run: |
55+
curl --fail --location --retry 3 \
56+
https://github.com/WebAssembly/wabt/releases/download/1.0.42/wabt-1.0.42-linux-x64.tar.gz \
57+
--output "${RUNNER_TEMP}/wabt.tar.gz"
58+
echo "84895407a6bbb80e918f33b16b2fb2206021c150b6bc9ff6f761263a745ab131 ${RUNNER_TEMP}/wabt.tar.gz" | sha256sum --check
59+
tar -xzf "${RUNNER_TEMP}/wabt.tar.gz" -C "${RUNNER_TEMP}"
60+
echo "${RUNNER_TEMP}/wabt-1.0.42/bin" >> "${GITHUB_PATH}"
61+
- name: Install pinned Rust and WASI target
62+
run: rustup toolchain install 1.90.0 --profile minimal --target wasm32-wasip1
63+
- name: Check WAT binaries and test rebuilt Rust guests
64+
run: python3 .github/scripts/verify-wasm-fixtures.py

‎RELEASE-NOTES.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,33 @@
22

33
### API Changes
44

5+
- `MemorySafeLRUMap` is deprecated since 2.7.2. The class, public constructors,
6+
serialization identifier and existing behavior are retained for downstream
7+
source and binary compatibility. No removal release is scheduled; a future
8+
removal requires a public API compatibility review and advance release notice.
9+
For caches accessed through the `Map` API, consider the existing
10+
`WindowTinyLFUMap` with an explicit maximum entry count, for example
11+
`new WindowTinyLFUMap<>(maximumEntries)`. Choose that count for the application's
12+
working set: the old `maxFreeMemory` argument is a free-memory threshold in
13+
bytes, not an entry count. Review the different eviction policy, non-null
14+
key/value requirements and absence of `LRUMap`-specific operations. Code that
15+
relies on memory-pressure eviction should retain `MemorySafeLRUMap` until an
16+
equivalent policy is chosen; `WindowTinyLFUMap` does not implement that
17+
threshold. (#7444)
18+
519
- `DiscoveryUpstreamDataSubscriber#unSubscribe(DiscoverySyncData)` has been replaced by `unSubscribe(DiscoveryUpstreamKey)`. Downstream implementations must update their method signature; this is a source- and binary-incompatible change. (#7289)
620

721
### Behavior Changes
822

23+
- `AbstractShenyuWasmPlugin()` now selects the JSON request/configuration/result
24+
bridge rather than no-op host functions. Results require versioned UTF-8 JSON
25+
within 1 MiB; invalid results fail the request with HTTP 500. Subclasses that
26+
require the former no-op callbacks can call `super((Consumer<Store>) null)`;
27+
the Consumer constructor retains custom adapters and existing overrides.
28+
The default Java continuation hook now delegates to the chain and argument
29+
IDs are generated automatically. Register a bean destroy method or call
30+
`close()` explicitly for shutdown cleanup; no starter wiring is added. (#7561)
31+
932
- HTTP client registration now treats non-success Admin responses as failures,
1033
using the same bounded failback policy as transport errors (18 retry attempts
1134
per pending task). The text response contract does not reliably distinguish

‎shenyu-common/src/main/java/org/apache/shenyu/common/cache/MemorySafeLRUMap.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,13 @@
2828
* is that it handles memory issues via {@link org.apache.shenyu.common.concurrent.MemoryLimitCalculator}.
2929
*
3030
* <p>Note: this class is not thread safe! see https://github.com/apache/shenyu/issues/3929
31+
*
32+
* @deprecated since 2.7.2, retained for downstream compatibility. For bounded caches accessed through
33+
* the {@link Map} API, consider {@link WindowTinyLFUMap} with an explicit maximum entry count.
34+
* Review its different eviction policy and lack of this class's free-memory threshold
35+
* and {@link LRUMap}-specific operations before migrating.
3136
*/
37+
@Deprecated(since = "2.7.2")
3238
@NotThreadSafe
3339
public class MemorySafeLRUMap<K, V> extends LRUMap<K, V> {
3440

‎shenyu-plugin/shenyu-plugin-ai/shenyu-plugin-ai-request-transformer/src/main/java/org/apache/shenyu/plugin/ai/transformer/request/AiRequestTransformerPlugin.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import java.util.Map;
5959
import java.util.Objects;
6060
import java.util.Optional;
61+
import java.util.regex.Pattern;
6162
import java.util.stream.Collectors;
6263
import java.util.stream.Stream;
6364

@@ -68,16 +69,18 @@ public class AiRequestTransformerPlugin extends AbstractShenyuPlugin {
6869

6970
private static final Logger LOG = LoggerFactory.getLogger(AiRequestTransformerPlugin.class);
7071

72+
private static final Pattern REQUEST_LINE_PATTERN = Pattern.compile("^(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD)\\s.*\\sHTTP/1.1$");
73+
7174
/**
7275
* Valid characters of a generated origin-form path component.
7376
*/
74-
private static final String PATH_PATTERN = "^/[a-zA-Z0-9/_\\-]*$";
77+
private static final Pattern REQUEST_PATH_PATTERN = Pattern.compile("^/[a-zA-Z0-9/_\\-]*$");
7578

7679
/**
7780
* Valid characters of a generated query component: RFC 3986 query syntax (percent-encoded
7881
* sequences must be well formed so that the target is always a legal URI reference).
7982
*/
80-
private static final String QUERY_PATTERN = "^(%[0-9a-fA-F]{2}|[a-zA-Z0-9\\-._~!$&'()*+,;=:@/?])*$";
83+
private static final Pattern QUERY_PATTERN = Pattern.compile("^(%[0-9a-fA-F]{2}|[a-zA-Z0-9\\-._~!$&'()*+,;=:@/?])*$");
8184

8285
private final List<HttpMessageReader<?>> messageReaders;
8386

@@ -267,12 +270,12 @@ private static Mono<ServerWebExchange> rewriteRequestPath(final ServerWebExchang
267270
return Mono.just(exchange);
268271
}
269272

270-
if (!newPath.matches(PATH_PATTERN)) {
273+
if (!REQUEST_PATH_PATTERN.matcher(newPath).matches()) {
271274
LOG.warn("Extracted path contains invalid characters: {}, Will continue to use the original path.", newPath);
272275
return Mono.just(exchange);
273276
}
274277

275-
if (Objects.nonNull(newQuery) && !newQuery.matches(QUERY_PATTERN)) {
278+
if (Objects.nonNull(newQuery) && !QUERY_PATTERN.matcher(newQuery).matches()) {
276279
LOG.warn("Extracted query contains invalid characters: {}, Will continue to use the original path.", newQuery);
277280
return Mono.just(exchange);
278281
}
@@ -320,7 +323,7 @@ static HttpHeaders extractHeadersFromAiResponse(final String aiResponse) {
320323
boolean headerSectionStarted = false;
321324
while (Objects.nonNull(line = reader.readLine())) {
322325
if (!headerSectionStarted) {
323-
if (line.startsWith("HTTP/1.1") || line.matches("^(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD)\\s.*\\sHTTP/1.1$")) {
326+
if (line.startsWith("HTTP/1.1") || REQUEST_LINE_PATTERN.matcher(line).matches()) {
324327
headerSectionStarted = true;
325328
continue;
326329
}
@@ -374,4 +377,3 @@ public String named() {
374377
}
375378

376379
}
377-

‎shenyu-plugin/shenyu-plugin-ai/shenyu-plugin-ai-response-transformer/src/main/java/org/apache/shenyu/plugin/ai/transformer/response/AiResponseTransformerPlugin.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import java.util.List;
5959
import java.util.Objects;
6060
import java.util.Optional;
61+
import java.util.regex.Pattern;
6162
import java.util.stream.Collectors;
6263
import java.util.stream.Stream;
6364
import java.util.zip.GZIPInputStream;
@@ -74,6 +75,8 @@ public class AiResponseTransformerPlugin extends AbstractShenyuPlugin {
7475

7576
private static final Logger LOG = LoggerFactory.getLogger(AiResponseTransformerPlugin.class);
7677

78+
private static final Pattern REQUEST_LINE_PATTERN = Pattern.compile("^(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD)\\s.*\\sHTTP/1.1$");
79+
7780
private static final ObjectMapper MAPPER = new ObjectMapper();
7881

7982
private final List<HttpMessageReader<?>> messageReaders;
@@ -263,7 +266,7 @@ static HttpHeaders extractHeadersFromAiResponse(final String aiResponse) {
263266
boolean headerSectionStarted = false;
264267
while (Objects.nonNull(line = reader.readLine())) {
265268
if (!headerSectionStarted) {
266-
if (line.startsWith("HTTP/1.1") || line.matches("^(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD)\\s.*\\sHTTP/1.1$")) {
269+
if (line.startsWith("HTTP/1.1") || REQUEST_LINE_PATTERN.matcher(line).matches()) {
267270
headerSectionStarted = true;
268271
continue;
269272
}

‎shenyu-plugin/shenyu-plugin-wasm-api/src/main/java/org/apache/shenyu/plugin/wasm/api/loader/WasmLoader.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,7 @@
2525
import com.dylibso.chicory.runtime.Store;
2626
import com.dylibso.chicory.wasm.Parser;
2727
import com.dylibso.chicory.wasm.WasmModule;
28-
import com.dylibso.chicory.wasm.types.Export;
2928
import com.dylibso.chicory.wasm.types.ExportSection;
30-
import com.dylibso.chicory.wasm.types.ExternalType;
3129
import com.dylibso.chicory.wasm.types.FunctionType;
3230
import com.dylibso.chicory.wasm.types.ValType;
3331
import org.apache.shenyu.plugin.wasm.api.exception.ShenyuWasmException;
@@ -279,9 +277,8 @@ protected long onPutResult(final long argId, final long addr, final int len) {
279277
*/
280278
public Optional<ExportFunction> getWasmExtern(final String funcName) {
281279
ExportSection exports = instance.module().exportSection();
282-
for (int i = 0; i < exports.exportCount(); i++) {
283-
Export export = exports.getExport(i);
284-
if (export.name().equals(funcName) && export.exportType() == ExternalType.FUNCTION) {
280+
for (int index = 0; index < exports.exportCount(); index++) {
281+
if (funcName.equals(exports.getExport(index).name())) {
285282
return Optional.of(instance.export(funcName));
286283
}
287284
}

‎shenyu-plugin/shenyu-plugin-wasm-api/src/test/java/org/apache/shenyu/plugin/wasm/api/AbstractWasmPluginTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@
1818
package org.apache.shenyu.plugin.wasm.api;
1919

2020
import com.dylibso.chicory.runtime.Memory;
21+
import com.dylibso.chicory.wasm.InvalidException;
2122
import org.apache.shenyu.common.config.ShenyuConfig;
2223
import org.apache.shenyu.plugin.api.ShenyuPluginChain;
2324
import org.apache.shenyu.plugin.api.utils.SpringBeanUtils;
25+
import org.apache.shenyu.plugin.wasm.api.loader.WasmLoader;
2426
import org.junit.jupiter.api.BeforeEach;
2527
import org.junit.jupiter.api.Test;
2628
import org.springframework.context.ConfigurableApplicationContext;
@@ -35,6 +37,7 @@
3537
import java.util.concurrent.ConcurrentHashMap;
3638

3739
import static org.junit.jupiter.api.Assertions.assertEquals;
40+
import static org.junit.jupiter.api.Assertions.assertThrows;
3841
import static org.junit.jupiter.api.Assertions.assertTrue;
3942
import static org.mockito.Mockito.mock;
4043
import static org.mockito.Mockito.verify;
@@ -74,6 +77,21 @@ public void executeRustWasmPluginTest() {
7477
StepVerifier.create(rustPlugin.execute(exchange, shenyuPluginChain)).expectSubscription().verifyComplete();
7578
verify(shenyuPluginChain).execute(exchange);
7679
}
80+
81+
@Test
82+
public void missingExportReturnsEmptyTest() {
83+
try (WasmLoader loader = new WasmLoader(RustWasmPlugin.class, null)) {
84+
assertTrue(loader.getWasmExtern("missingCallback").isEmpty());
85+
assertTrue(loader.getWasmExtern("execute").isPresent());
86+
}
87+
}
88+
89+
@Test
90+
public void nonFunctionExportStillFailsTest() {
91+
try (WasmLoader loader = new WasmLoader(RustWasmPlugin.class, null)) {
92+
assertThrows(InvalidException.class, () -> loader.getWasmExtern("memory"));
93+
}
94+
}
7795

7896
private void mockShenyuConfig() {
7997
ConfigurableApplicationContext context = mock(ConfigurableApplicationContext.class);

‎shenyu-plugin/shenyu-plugin-wasm-api/src/test/java/org/apache/shenyu/plugin/wasm/api/WasmLoaderTest.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,16 @@
1717

1818
package org.apache.shenyu.plugin.wasm.api;
1919

20+
import com.dylibso.chicory.wasm.InvalidException;
2021
import org.apache.shenyu.plugin.wasm.api.loader.WasmLoader;
2122
import org.junit.jupiter.api.Test;
2223

2324
import static org.junit.jupiter.api.Assertions.assertFalse;
25+
import static org.junit.jupiter.api.Assertions.assertThrows;
2426
import static org.junit.jupiter.api.Assertions.assertTrue;
2527

2628
/**
27-
* Export lookup must distinguish callable exports from absent or non-function exports.
29+
* Export lookup treats missing exports as optional and rejects non-function exports.
2830
*/
2931
final class WasmLoaderTest {
3032

@@ -33,7 +35,7 @@ void lookupReturnsOnlyExportedFunctions() {
3335
try (WasmLoader loader = new WasmLoader(AbstractWasmPluginTest.RustWasmPlugin.class, null)) {
3436
assertTrue(loader.getWasmExtern("execute").isPresent());
3537
assertFalse(loader.getWasmExtern("does_not_exist").isPresent());
36-
assertFalse(loader.getWasmExtern("memory").isPresent());
38+
assertThrows(InvalidException.class, () -> loader.getWasmExtern("memory"));
3739
}
3840
}
3941
}

0 commit comments

Comments
 (0)