Submit form with programmatically added input field sends empty value #606
Description
For a test, I am using WebClient to download a static HTML page and click on the Submit button.
Before I submit, I am dynamically adding an input field to the form. In the server, I expect the extra field to be received. I can see the key is received, but the value is blank. If I append more than one input field, they all have blank values.
I am using the latest HtmlUnit 3.3.0. It is listed in Maven Central with date June 4, 2023:
<!-- https://mvnrepository.com/artifact/org.htmlunit/htmlunit -->
<dependency>
<groupId>org.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>3.3.0</version>
<scope>test</scope>
</dependency>
I tried to trace the issue in WebClient. I think the problem might be in SubmittableElement.getSubmitNameValuePairs(). For the static HTML form fields, it returns the name and value. For the dynamically added form field, it returns name with blank value.
If there is a workaround for the problem, please let me know. It would be much appreciated!
In the meantime, I created a unit test to reproduce and demonstrate the issue. It uses WebClient and MockWebServer. The request and response bodies are captured and printed for both requests (GET HTML page, and POST HTML form).
This is the input field I programmatically append to the form.
HtmlTextInput[<input type="hidden" name="appendedTextName" value="appendedTextValue">]
This is the POST request body received by the server. Notice appendedTextName at the end with an empty value, instead of the expected value.
staticTextName=staticTextValue&staticHiddenName=staticHiddenValue&staticCheckboxName=staticCheckboxValue&appendedTextName=
Here is the unit test.
package org.mypackage;
import lombok.extern.slf4j.Slf4j;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest;
import org.htmlunit.Page;
import org.htmlunit.WebClient;
import org.htmlunit.html.DomElement;
import org.htmlunit.html.HtmlElement;
import org.htmlunit.html.HtmlForm;
import org.htmlunit.html.HtmlPage;
import org.htmlunit.html.SubmittableElement;
import org.htmlunit.util.NameValuePair;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.http.HttpStatus;
import java.io.IOException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static java.util.Objects.requireNonNull;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
@Slf4j
public class HtmlInputBugReportTest {
private static final String STATIC_HTML_PAGE_WITH_FORM = """
<html>
<head><title>Static Page With Form</title></head>
<body>
<form name="staticForm" method="post" action="http://localhost:%PORT%/">
<input type="text" name="staticTextName" value="staticTextValue"/>
<input type="hidden" name="staticHiddenName" value="staticHiddenValue"/>
<input type="checkbox" name="staticCheckboxName" value="staticCheckboxValue" checked/>
<button type="submit">Submit Button</button>
</form>
</html>
""";
private static MockWebServer MOCK_WEB_SERVER;
@BeforeAll static void beforeAll() throws IOException {
MOCK_WEB_SERVER = new MockWebServer();
MOCK_WEB_SERVER.start();
}
@AfterAll static void afterAll() throws IOException {
MOCK_WEB_SERVER.shutdown();
}
@Test void appendNonEmptyInputValueToFormDoesNotWork() throws IOException, InterruptedException {
try (final WebClient webClient = new WebClient()) {
// enqueue GET response (i.e. static HTML page with a form)
final String staticHtmlPage = STATIC_HTML_PAGE_WITH_FORM.replace("%PORT%", Integer.toString(MOCK_WEB_SERVER.getPort()));
MOCK_WEB_SERVER.enqueue(new MockResponse().setResponseCode(200).setBody(staticHtmlPage));
// request the static HTML page
final HtmlPage htmlPage = webClient.getPage("http://localhost:" + MOCK_WEB_SERVER.getPort());
final RecordedRequest recordedGetRequest = MOCK_WEB_SERVER.takeRequest();
final String getRequestBody = recordedGetRequest.getBody().readUtf8(); // GET request body, as received by the server
final String getResponseBody = htmlPage.getBody().asXml(); // GET response body, as received by the client
log.info("Client's GET request body:\n{}", getRequestBody);
log.info("Client's GET response body:\n{}", getResponseBody);
assertThat(htmlPage.getWebResponse().getStatusCode()).isEqualTo(HttpStatus.OK.value()); // Expect HTTP 200 OK
assertThat(recordedGetRequest.getMethod()).isEqualTo("GET");
assertThat(recordedGetRequest.getHeader("Content-Type")).isNull();
assertThat(getRequestBody).isEmpty();
// dynamically append an input field to the form in the parsed static HTML page
final String appendedTextName = "appendedTextName";
final String appendedTextValue = "appendedTextValue";
final HtmlForm htmlForm = requireNonNull(htmlPage.getFormByName("staticForm"));
final DomElement domElement = htmlPage.createElement("input");
domElement.setAttribute("type", "hidden");
domElement.setAttribute("name", appendedTextName);
domElement.setAttribute("value", appendedTextValue);
htmlForm.appendChild(domElement);
log.info("Appended to the form: {}", domElement); // HtmlTextInput[<input type="hidden" name="appendedTextName" value="appendedTextValue">]
// before submit, check if the client dynamically appended the input field to the form with the expected name and value
final SubmittableElement submittableElement = (SubmittableElement) domElement;
final NameValuePair nameValuePair = submittableElement.getSubmitNameValuePairs()[0];
assertThat(nameValuePair.getName()).isEqualTo(appendedTextName);
// assertThat(nameValuePair.getValue()).isEqualTo(appendedTextValue); // Root cause of server receiving name with empty value
// enqueue POST response (i.e. content doesn't really matter)
MOCK_WEB_SERVER.enqueue(new MockResponse().setResponseCode(200).setBody("Received your request").addHeader("Content-Type", "text/plain; charset=UTF-8"));
// submit the form by clicking the Submit button
final List<HtmlElement> submitButton = requireNonNull(htmlForm.getElementsByAttribute("button", "type", "submit"));
final Page submitResponse = submitButton.get(0).click();
final RecordedRequest recordedPostRequest = MOCK_WEB_SERVER.takeRequest();
final String postRequestBody = recordedPostRequest.getBody().readUtf8(); // POST request body, as received by the server
final String postResponseBody = submitResponse.getWebResponse().getContentAsString(); // POST response body, as received by the client
log.info("Client's POST request body:\n\n{}\n", postRequestBody);
log.info("Client's POST response body:\n\n{}\n", postResponseBody);
assertThat(submitResponse.getWebResponse().getStatusCode()).isEqualTo(HttpStatus.OK.value()); // Expect HTTP 200 OK
assertThat(recordedPostRequest.getMethod()).isEqualTo("POST");
assertThat(recordedPostRequest.getHeader("Content-Type")).isEqualTo("application/x-www-form-urlencoded");
assertThat(postRequestBody).isNotEmpty();
// find the dynamically appended input field in the request body received by the server, and validate it has the expected value that the client intended to send
for (final String parameterKeyEqualsValue : postRequestBody.split("&")) {
final String[] parameterKeyAndValue = parameterKeyEqualsValue.split("=", 2);
final String name = URLDecoder.decode(parameterKeyAndValue[0], StandardCharsets.UTF_8);
final String value = URLDecoder.decode(parameterKeyAndValue[1], StandardCharsets.UTF_8);
if (name.equals(appendedTextName)) {
assertThat(value).isEqualTo(appendedTextValue); // FAILS: Server received the expected name but not the value, because the client sent an empty value
}
}
}
}
}