Skip to content

Commit 8a76f78

Browse files
authored
Support Qwen/QwQ-32B (needed str.{split,lstrip}) (#56)
* Add str.split(sep), .strip([chars]), .rstrip([chars]), .lstrip([chars]) * test Qwen/QwQ-32B template
1 parent dee1b89 commit 8a76f78

File tree

3 files changed

+45
-5
lines changed

3 files changed

+45
-5
lines changed

include/minja/minja.hpp

+37-5
Original file line numberDiff line numberDiff line change
@@ -1379,13 +1379,27 @@ struct ArgumentsExpression {
13791379
}
13801380
};
13811381

1382-
static std::string strip(const std::string & s) {
1383-
auto start = s.find_first_not_of(" \t\n\r");
1382+
static std::string strip(const std::string & s, const std::string & chars = "", bool left = true, bool right = true) {
1383+
auto charset = chars.empty() ? " \t\n\r" : chars;
1384+
auto start = left ? s.find_first_not_of(charset) : 0;
13841385
if (start == std::string::npos) return "";
1385-
auto end = s.find_last_not_of(" \t\n\r");
1386+
auto end = right ? s.find_last_not_of(charset) : s.size() - 1;
13861387
return s.substr(start, end - start + 1);
13871388
}
13881389

1390+
static std::vector<std::string> split(const std::string & s, const std::string & sep) {
1391+
std::vector<std::string> result;
1392+
size_t start = 0;
1393+
size_t end = s.find(sep);
1394+
while (end != std::string::npos) {
1395+
result.push_back(s.substr(start, end - start));
1396+
start = end + sep.length();
1397+
end = s.find(sep, start);
1398+
}
1399+
result.push_back(s.substr(start));
1400+
return result;
1401+
}
1402+
13891403
static std::string capitalize(const std::string & s) {
13901404
if (s.empty()) return s;
13911405
auto result = s;
@@ -1468,8 +1482,26 @@ class MethodCallExpr : public Expression {
14681482
} else if (obj.is_string()) {
14691483
auto str = obj.get<std::string>();
14701484
if (method->get_name() == "strip") {
1471-
vargs.expectArgs("strip method", {0, 0}, {0, 0});
1472-
return Value(strip(str));
1485+
vargs.expectArgs("strip method", {0, 1}, {0, 0});
1486+
auto chars = vargs.args.empty() ? "" : vargs.args[0].get<std::string>();
1487+
return Value(strip(str, chars));
1488+
} else if (method->get_name() == "lstrip") {
1489+
vargs.expectArgs("lstrip method", {0, 1}, {0, 0});
1490+
auto chars = vargs.args.empty() ? "" : vargs.args[0].get<std::string>();
1491+
return Value(strip(str, chars, /* left= */ true, /* right= */ false));
1492+
} else if (method->get_name() == "rstrip") {
1493+
vargs.expectArgs("rstrip method", {0, 1}, {0, 0});
1494+
auto chars = vargs.args.empty() ? "" : vargs.args[0].get<std::string>();
1495+
return Value(strip(str, chars, /* left= */ false, /* right= */ true));
1496+
} else if (method->get_name() == "split") {
1497+
vargs.expectArgs("split method", {1, 1}, {0, 0});
1498+
auto sep = vargs.args[0].get<std::string>();
1499+
auto parts = split(str, sep);
1500+
Value result = Value::array();
1501+
for (const auto& part : parts) {
1502+
result.push_back(Value(part));
1503+
}
1504+
return result;
14731505
} else if (method->get_name() == "capitalize") {
14741506
vargs.expectArgs("capitalize method", {0, 0}, {0, 0});
14751507
return Value(capitalize(str));

tests/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,7 @@ set(MODEL_IDS
263263
Qwen/Qwen2.5-VL-3B-Instruct
264264
Qwen/Qwen2.5-VL-72B-Instruct
265265
Qwen/Qwen2.5-VL-7B-Instruct
266+
Qwen/QwQ-32B
266267
Qwen/QwQ-32B-Preview
267268
rubenroy/Zurich-14B-GCv2-5m
268269
rubenroy/Zurich-7B-GCv2-5m

tests/test-syntax.cpp

+7
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,13 @@ TEST(SyntaxTest, SimpleCases) {
7474
return testing::Throws<std::runtime_error>(Property(&std::runtime_error::what, testing::HasSubstr(expected_substr)));
7575
};
7676

77+
EXPECT_EQ("a", render("{{ ' a '.strip() }}", {}, {}));
78+
EXPECT_EQ("a ", render("{{ ' a '.lstrip() }}", {}, {}));
79+
EXPECT_EQ(" a", render("{{ ' a '.rstrip() }}", {}, {}));
80+
EXPECT_EQ("bcXYZab", render("{{ 'abcXYZabc'.strip('ac') }}", {}, {}));
81+
82+
EXPECT_EQ(R"(["a", "b"])", render("{{ 'a b'.split(' ') | tojson }}", {}, {}));
83+
7784
EXPECT_EQ(
7885
"Ok",
7986
render("{{ 'ok'.capitalize() }}", {}, {}));

0 commit comments

Comments
 (0)