Kohei Nozaki's blog 

Environment

• WireMock 1.4.3
• HttpClient 4.3.2
• Oracle JDK7u51

Why need it?

• Sometimes website scraping stops at unexpected error like socket timeout or internal server error.
• We prefer to retry several times at such occasion.
• So we have to develop some retrying mechanism, and we have to create a test and create the mock of server.
  • So in this article, I'm going to try to create the mock with WireMock.

Requirements for client

• Retry if processing fails, for 3 times.
• Retry when timeout occurred.
  • SocketTimeoutException
• Retry when server returned status code 5xx

Test cases

1. Server returns code 200 without any problems.
2. Server returns code 500 at first, then client will retry, server returns code 200.
3. Server returns code 500 forever, then client will retry 3 times and give up.
4. Server delays the response, then client will regard as timeout and retry, then server returns code 200.
5. Server delays the response forever, then client will retry 3 times and give up.

Resources

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>org.nailedtothex</groupId>
    <artifactId>wiremock</artifactId>
    <version>0.0.1-SNAPSHOT</version>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.7</maven.compiler.source>
        <maven.compiler.target>1.7</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.11</version>
            <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.hamcrest</groupId>
          <artifactId>hamcrest-core</artifactId>
          <version>1.3</version>
          <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>com.github.tomakehurst</groupId>
            <artifactId>wiremock</artifactId>
            <version>1.43</version>
            <classifier>standalone</classifier>
            <scope>test</scope>
            <exclusions>
                <exclusion>
                    <groupId>*</groupId>
                    <artifactId>*</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.3.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>fluent-hc</artifactId>
            <version>4.3.2</version>
        </dependency>
    </dependencies>

</project>

RetryableHttpFetcher.java

package org.nailedtothex.wiremock;

import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.ServiceUnavailableRetryStrategy;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

public class RetryableHttpFetcher {

    private static final Logger log = Logger.getLogger(RetryableHttpFetcher.class.getName());

    // these parameters would be better to retrieve through JNDI or any other mechanism
    private int MAX_RETRIES = 3;
    private int RETRY_INTERVAL = 1000;
    private int TIMEOUT = 500;

    private final ServiceUnavailableRetryStrategy MY_SERVICE_UNAVAILABLE_RETRY_STRATEGY = new ServiceUnavailableRetryStrategy() {
        @Override
        public boolean retryRequest(HttpResponse response, int executionCount,
                org.apache.http.protocol.HttpContext context) {

            boolean rc = response.getStatusLine().getStatusCode() >= 500 && executionCount <= MAX_RETRIES;

            log.log(Level.INFO,
                    "retryRequest(): returning={0}, statusCode={1}, executionCount={2}, maxRetries={3}, interval={4}",
                    new Object[] { rc, response.getStatusLine().getStatusCode(), executionCount, MAX_RETRIES,
                            RETRY_INTERVAL });

            return rc;
        }

        @Override
        public long getRetryInterval() {
            return RETRY_INTERVAL;
        }
    };

    private final HttpRequestRetryHandler MY_HTTP_REQUEST_RETRY_HANDLER = new HttpRequestRetryHandler() {

        @Override
        public boolean retryRequest(IOException e, int executionCount, HttpContext context) {
            log.log(Level.INFO, "retryRequest(): exception={0}, executionCount={1}, maxRetries={2}",
                    new Object[] { e.getClass(), executionCount, MAX_RETRIES });

            if (executionCount > MAX_RETRIES) {
                log.log(Level.INFO, "give up: {0}", executionCount);
                return false;
            }

            if (e instanceof java.net.SocketTimeoutException) {
                log.log(Level.INFO, "retry: {0}", e.getMessage());
                return true;
            }

            log.log(Level.INFO, "not retry: {0}", e.getMessage());
            return false;
        }
    };

    private final RequestConfig MY_REQUEST_CONFIG = RequestConfig.custom()
            .setConnectionRequestTimeout(TIMEOUT)
            .setConnectTimeout(TIMEOUT)
            .setSocketTimeout(TIMEOUT)
            .build();

    public String fetchAsString(String url) throws ClientProtocolException, IOException {

        try (CloseableHttpClient client = HttpClientBuilder.create()
                .setDefaultRequestConfig(MY_REQUEST_CONFIG)
                .setRetryHandler(MY_HTTP_REQUEST_RETRY_HANDLER)
                .setServiceUnavailableRetryStrategy(MY_SERVICE_UNAVAILABLE_RETRY_STRATEGY)
                .build()) {

            try (CloseableHttpResponse res = client.execute(new HttpGet(url))) {
                if (res.getStatusLine().getStatusCode() >= 400) {
                    throw new HttpResponseException(res.getStatusLine().getStatusCode(), res.getStatusLine()
                            .getReasonPhrase());
                }
                return EntityUtils.toString(res.getEntity());
            }
        }

    }
}

RetryableHttpFetcherTest.java

package org.nailedtothex.wiremock;

import static com.github.tomakehurst.wiremock.client.WireMock.*;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.*;

import java.net.SocketTimeoutException;

import org.apache.http.client.HttpResponseException;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.github.tomakehurst.wiremock.stubbing.Scenario;

public class RetryableHttpFetcherTest {

    @Rule
    public WireMockRule wireMockRule = new WireMockRule(18089);

    private RetryableHttpFetcher instance;

    @Before
    public void init() {
        instance = new RetryableHttpFetcher();
    }

    @Test
    public void test1_ok() throws Exception {
        stubFor(get(urlEqualTo("/hoge.txt")).willReturn(
                aResponse().withStatus(200).withHeader("Content-Type", "text/plain").withBody("OK")));
        String expected = "OK";

        String actual = instance.fetchAsString("http://localhost:18089/hoge.txt");

        assertThat(actual, is(expected));
    }

    @Test
    public void test2_retryAt500() throws Exception {
        stubFor(get(urlEqualTo("/500")).inScenario("retry at 500")
                .whenScenarioStateIs(Scenario.STARTED)
                .willSetStateTo("one time requested")
                .willReturn(aResponse().withBody("error").withStatus(500)));

        stubFor(get(urlEqualTo("/500")).inScenario("retry at 500")
                .whenScenarioStateIs("one time requested")
                .willReturn(aResponse().withBody("OK").withStatus(200)));

        String actual = instance.fetchAsString("http://localhost:18089/500");

        assertThat(actual, is("OK"));
    }

    @Test(expected = HttpResponseException.class)
    public void test3_retryAt500GiveUp() throws Exception {
        stubFor(get(urlEqualTo("/500"))
                .willReturn(aResponse().withBody("500").withStatus(500)));

        instance.fetchAsString("http://localhost:18089/500");
    }

    @Test
    public void test4_retryAtTimeout() throws Exception {
        stubFor(get(urlEqualTo("/timeout")).inScenario("retrying")
                .whenScenarioStateIs(Scenario.STARTED)
                .willSetStateTo("one time requested")
                .willReturn(aResponse().withBody("error").withStatus(500).withFixedDelay(3000)));
        stubFor(get(urlEqualTo("/timeout")).inScenario("retrying")
                .whenScenarioStateIs("one time requested")
                .willReturn(aResponse().withBody("OK").withStatus(200)));

        String actual = instance.fetchAsString("http://localhost:18089/timeout");
        assertThat(actual, is("OK"));
    }

    @Test(expected = SocketTimeoutException.class)
    public void test5_retryAtTimeoutGiveUp() throws Exception {
        stubFor(get(urlEqualTo("/timeout"))
                .willReturn(aResponse().withBody("timeout").withStatus(500).withFixedDelay(Integer.MAX_VALUE)));
        instance.fetchAsString("http://localhost:18089/timeout");
    }

    @Test(expected = HttpResponseException.class)
    public void notFound() throws Exception {
        instance.fetchAsString("http://localhost:18089/NOT_FOUND");
    }
}

Test logs

• All tests were passed.

test1_ok

2 26, 2014 11:34:22 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=false, statusCode=200, executionCount=1, maxRetries=3, interval=1,000

test2_retryAt500

2 26, 2014 11:35:32 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=true, statusCode=500, executionCount=1, maxRetries=3, interval=1,000
2 26, 2014 11:35:33 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=false, statusCode=200, executionCount=2, maxRetries=3, interval=1,000

test3_retryAt500GiveUp

2 26, 2014 11:35:51 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=true, statusCode=500, executionCount=1, maxRetries=3, interval=1,000
2 26, 2014 11:35:52 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=true, statusCode=500, executionCount=2, maxRetries=3, interval=1,000
2 26, 2014 11:35:53 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=true, statusCode=500, executionCount=3, maxRetries=3, interval=1,000
2 26, 2014 11:35:54 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=false, statusCode=500, executionCount=4, maxRetries=3, interval=1,000

test4_retryAtTimeout

2 26, 2014 11:36:12 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retryRequest(): exception=class java.net.SocketTimeoutException, executionCount=1, maxRetries=3
2 26, 2014 11:36:12 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retry: Read timed out
2 26, 2014 11:36:12 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$1 retryRequest
情報: retryRequest(): returning=false, statusCode=200, executionCount=1, maxRetries=3, interval=1,000

test5_retryAtTimeoutGiveUp

2 26, 2014 11:36:39 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retryRequest(): exception=class java.net.SocketTimeoutException, executionCount=1, maxRetries=3
2 26, 2014 11:36:39 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retry: Read timed out
2 26, 2014 11:36:40 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retryRequest(): exception=class java.net.SocketTimeoutException, executionCount=2, maxRetries=3
2 26, 2014 11:36:40 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retry: Read timed out
2 26, 2014 11:36:40 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retryRequest(): exception=class java.net.SocketTimeoutException, executionCount=3, maxRetries=3
2 26, 2014 11:36:40 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retry: Read timed out
2 26, 2014 11:36:41 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: retryRequest(): exception=class java.net.SocketTimeoutException, executionCount=4, maxRetries=3
2 26, 2014 11:36:41 午前 org.nailedtothex.wiremock.RetryableHttpFetcher$2 retryRequest
情報: give up: 4

References

1. HttpClient 4 Cookbook