1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27 package org.apache.http.impl.client.cache;
28
29 import java.io.IOException;
30 import java.lang.reflect.UndeclaredThrowableException;
31 import java.net.URI;
32 import java.util.Collections;
33 import java.util.Date;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Set;
39 import java.util.concurrent.ArrayBlockingQueue;
40 import java.util.concurrent.ExecutorService;
41 import java.util.concurrent.RejectedExecutionException;
42 import java.util.concurrent.ThreadPoolExecutor;
43 import java.util.concurrent.TimeUnit;
44 import java.util.concurrent.atomic.AtomicLong;
45
46 import org.apache.commons.logging.Log;
47 import org.apache.commons.logging.LogFactory;
48 import org.apache.http.Header;
49 import org.apache.http.HeaderElement;
50 import org.apache.http.HttpEntity;
51 import org.apache.http.HttpHost;
52 import org.apache.http.HttpMessage;
53 import org.apache.http.HttpRequest;
54 import org.apache.http.HttpResponse;
55 import org.apache.http.HttpStatus;
56 import org.apache.http.HttpVersion;
57 import org.apache.http.ProtocolException;
58 import org.apache.http.ProtocolVersion;
59 import org.apache.http.RequestLine;
60 import org.apache.http.annotation.Contract;
61 import org.apache.http.annotation.ThreadingBehavior;
62 import org.apache.http.client.ClientProtocolException;
63 import org.apache.http.client.HttpClient;
64 import org.apache.http.client.ResponseHandler;
65 import org.apache.http.client.cache.CacheResponseStatus;
66 import org.apache.http.client.cache.HeaderConstants;
67 import org.apache.http.client.cache.HttpCacheEntry;
68 import org.apache.http.client.cache.HttpCacheStorage;
69 import org.apache.http.client.cache.ResourceFactory;
70 import org.apache.http.client.methods.HttpRequestWrapper;
71 import org.apache.http.client.methods.HttpUriRequest;
72 import org.apache.http.conn.ClientConnectionManager;
73 import org.apache.http.impl.client.DefaultHttpClient;
74 import org.apache.http.impl.cookie.DateParseException;
75 import org.apache.http.impl.cookie.DateUtils;
76 import org.apache.http.message.BasicHttpResponse;
77 import org.apache.http.params.HttpParams;
78 import org.apache.http.protocol.ExecutionContext;
79 import org.apache.http.protocol.HTTP;
80 import org.apache.http.protocol.HttpContext;
81 import org.apache.http.util.Args;
82 import org.apache.http.util.VersionInfo;
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129 @Deprecated
130 @Contract(threading = ThreadingBehavior.SAFE_CONDITIONAL)
131 public class CachingHttpClient implements HttpClient {
132
133
134
135
136
137
138
139 public static final String CACHE_RESPONSE_STATUS = "http.cache.response.status";
140
141 private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
142
143 private final AtomicLong cacheHits = new AtomicLong();
144 private final AtomicLong cacheMisses = new AtomicLong();
145 private final AtomicLong cacheUpdates = new AtomicLong();
146
147 private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
148
149 private final HttpClient backend;
150 private final HttpCache responseCache;
151 private final CacheValidityPolicy validityPolicy;
152 private final ResponseCachingPolicy responseCachingPolicy;
153 private final CachedHttpResponseGenerator responseGenerator;
154 private final CacheableRequestPolicy cacheableRequestPolicy;
155 private final CachedResponseSuitabilityChecker suitabilityChecker;
156
157 private final ConditionalRequestBuilder conditionalRequestBuilder;
158
159 private final long maxObjectSizeBytes;
160 private final boolean sharedCache;
161
162 private final ResponseProtocolCompliance responseCompliance;
163 private final RequestProtocolCompliance requestCompliance;
164
165 private final AsynchronousValidator asynchRevalidator;
166
167 private final Log log = LogFactory.getLog(getClass());
168
169 CachingHttpClient(
170 final HttpClient client,
171 final HttpCache cache,
172 final CacheConfig config) {
173 super();
174 Args.notNull(client, "HttpClient");
175 Args.notNull(cache, "HttpCache");
176 Args.notNull(config, "CacheConfig");
177 this.maxObjectSizeBytes = config.getMaxObjectSize();
178 this.sharedCache = config.isSharedCache();
179 this.backend = client;
180 this.responseCache = cache;
181 this.validityPolicy = new CacheValidityPolicy();
182 this.responseCachingPolicy = new ResponseCachingPolicy(maxObjectSizeBytes, sharedCache,
183 config.isNeverCacheHTTP10ResponsesWithQuery(), config.is303CachingEnabled());
184 this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
185 this.cacheableRequestPolicy = new CacheableRequestPolicy();
186 this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, config);
187 this.conditionalRequestBuilder = new ConditionalRequestBuilder();
188
189 this.responseCompliance = new ResponseProtocolCompliance();
190 this.requestCompliance = new RequestProtocolCompliance(config.isWeakETagOnPutDeleteAllowed());
191
192 this.asynchRevalidator = makeAsynchronousValidator(config);
193 }
194
195
196
197
198
199
200 public CachingHttpClient() {
201 this(new DefaultHttpClient(),
202 new BasicHttpCache(),
203 new CacheConfig());
204 }
205
206
207
208
209
210
211
212 public CachingHttpClient(final CacheConfig config) {
213 this(new DefaultHttpClient(),
214 new BasicHttpCache(config),
215 config);
216 }
217
218
219
220
221
222
223
224 public CachingHttpClient(final HttpClient client) {
225 this(client,
226 new BasicHttpCache(),
227 new CacheConfig());
228 }
229
230
231
232
233
234
235
236
237 public CachingHttpClient(final HttpClient client, final CacheConfig config) {
238 this(client,
239 new BasicHttpCache(config),
240 config);
241 }
242
243
244
245
246
247
248
249
250
251
252
253 public CachingHttpClient(
254 final HttpClient client,
255 final ResourceFactory resourceFactory,
256 final HttpCacheStorage storage,
257 final CacheConfig config) {
258 this(client,
259 new BasicHttpCache(resourceFactory, storage, config),
260 config);
261 }
262
263
264
265
266
267
268
269
270
271 public CachingHttpClient(
272 final HttpClient client,
273 final HttpCacheStorage storage,
274 final CacheConfig config) {
275 this(client,
276 new BasicHttpCache(new HeapResourceFactory(), storage, config),
277 config);
278 }
279
280 CachingHttpClient(
281 final HttpClient backend,
282 final CacheValidityPolicy validityPolicy,
283 final ResponseCachingPolicy responseCachingPolicy,
284 final HttpCache responseCache,
285 final CachedHttpResponseGenerator responseGenerator,
286 final CacheableRequestPolicy cacheableRequestPolicy,
287 final CachedResponseSuitabilityChecker suitabilityChecker,
288 final ConditionalRequestBuilder conditionalRequestBuilder,
289 final ResponseProtocolCompliance responseCompliance,
290 final RequestProtocolCompliance requestCompliance) {
291 final CacheConfig/cache/CacheConfig.html#CacheConfig">CacheConfig config = new CacheConfig();
292 this.maxObjectSizeBytes = config.getMaxObjectSize();
293 this.sharedCache = config.isSharedCache();
294 this.backend = backend;
295 this.validityPolicy = validityPolicy;
296 this.responseCachingPolicy = responseCachingPolicy;
297 this.responseCache = responseCache;
298 this.responseGenerator = responseGenerator;
299 this.cacheableRequestPolicy = cacheableRequestPolicy;
300 this.suitabilityChecker = suitabilityChecker;
301 this.conditionalRequestBuilder = conditionalRequestBuilder;
302 this.responseCompliance = responseCompliance;
303 this.requestCompliance = requestCompliance;
304 this.asynchRevalidator = makeAsynchronousValidator(config);
305 }
306
307 private AsynchronousValidator makeAsynchronousValidator(
308 final CacheConfig config) {
309 if (config.getAsynchronousWorkersMax() > 0) {
310 return new AsynchronousValidator(this, config);
311 }
312 return null;
313 }
314
315
316
317
318
319
320 public long getCacheHits() {
321 return cacheHits.get();
322 }
323
324
325
326
327
328
329 public long getCacheMisses() {
330 return cacheMisses.get();
331 }
332
333
334
335
336
337
338 public long getCacheUpdates() {
339 return cacheUpdates.get();
340 }
341
342 @Override
343 public HttpResponse execute(final HttpHost target, final HttpRequest request) throws IOException {
344 final HttpContext defaultContext = null;
345 return execute(target, request, defaultContext);
346 }
347
348 @Override
349 public <T> T execute(final HttpHost target, final HttpRequest request,
350 final ResponseHandler<? extends T> responseHandler) throws IOException {
351 return execute(target, request, responseHandler, null);
352 }
353
354 @Override
355 public <T> T execute(final HttpHost target, final HttpRequest request,
356 final ResponseHandler<? extends T> responseHandler, final HttpContext context) throws IOException {
357 final HttpResponse resp = execute(target, request, context);
358 return handleAndConsume(responseHandler,resp);
359 }
360
361 @Override
362 public HttpResponse execute(final HttpUriRequest request) throws IOException {
363 final HttpContext context = null;
364 return execute(request, context);
365 }
366
367 @Override
368 public HttpResponse execute(final HttpUriRequest request, final HttpContext context) throws IOException {
369 final URI uri = request.getURI();
370 final HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
371 return execute(httpHost, request, context);
372 }
373
374 @Override
375 public <T> T execute(final HttpUriRequest request, final ResponseHandler<? extends T> responseHandler)
376 throws IOException {
377 return execute(request, responseHandler, null);
378 }
379
380 @Override
381 public <T> T execute(final HttpUriRequest request, final ResponseHandler<? extends T> responseHandler,
382 final HttpContext context) throws IOException {
383 final HttpResponse resp = execute(request, context);
384 return handleAndConsume(responseHandler, resp);
385 }
386
387 private <T> T handleAndConsume(
388 final ResponseHandler<? extends T> responseHandler,
389 final HttpResponse response) throws Error, IOException {
390 final T result;
391 try {
392 result = responseHandler.handleResponse(response);
393 } catch (final Exception t) {
394 final HttpEntity entity = response.getEntity();
395 try {
396 IOUtils.consume(entity);
397 } catch (final Exception t2) {
398
399
400 this.log.warn("Error consuming content after an exception.", t2);
401 }
402 if (t instanceof RuntimeException) {
403 throw (RuntimeException) t;
404 }
405 if (t instanceof IOException) {
406 throw (IOException) t;
407 }
408 throw new UndeclaredThrowableException(t);
409 }
410
411
412
413 final HttpEntity entity = response.getEntity();
414 IOUtils.consume(entity);
415 return result;
416 }
417
418 @Override
419 public ClientConnectionManager getConnectionManager() {
420 return backend.getConnectionManager();
421 }
422
423 @Override
424 public HttpParams getParams() {
425 return backend.getParams();
426 }
427
428 @Override
429 public HttpResponse execute(final HttpHost target, final HttpRequest originalRequest, final HttpContext context)
430 throws IOException {
431
432 final HttpRequestWrapper request;
433 if (originalRequest instanceof HttpRequestWrapper) {
434 request = ((HttpRequestWrapper) originalRequest);
435 } else {
436 request = HttpRequestWrapper.wrap(originalRequest);
437 }
438 final String via = generateViaHeader(originalRequest);
439
440
441 setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
442
443 if (clientRequestsOurOptions(request)) {
444 setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
445 return new OptionsHttp11Response();
446 }
447
448 final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(
449 request, context);
450 if (fatalErrorResponse != null) {
451 return fatalErrorResponse;
452 }
453
454 requestCompliance.makeRequestCompliant(request);
455 request.addHeader("Via",via);
456
457 flushEntriesInvalidatedByRequest(target, request);
458
459 if (!cacheableRequestPolicy.isServableFromCache(request)) {
460 log.debug("Request is not servable from cache");
461 return callBackend(target, request, context);
462 }
463
464 final HttpCacheEntry entry = satisfyFromCache(target, request);
465 if (entry == null) {
466 log.debug("Cache miss");
467 return handleCacheMiss(target, request, context);
468 }
469
470 return handleCacheHit(target, request, context, entry);
471 }
472
473 private HttpResponse handleCacheHit(final HttpHost target, final HttpRequestWrapper request,
474 final HttpContext context, final HttpCacheEntry entry)
475 throws ClientProtocolException, IOException {
476 recordCacheHit(target, request);
477 HttpResponse out = null;
478 final Date now = getCurrentDate();
479 if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
480 log.debug("Cache hit");
481 out = generateCachedResponse(request, context, entry, now);
482 } else if (!mayCallBackend(request)) {
483 log.debug("Cache entry not suitable but only-if-cached requested");
484 out = generateGatewayTimeout(context);
485 } else {
486 log.debug("Revalidating cache entry");
487 return revalidateCacheEntry(target, request, context, entry, now);
488 }
489 if (context != null) {
490 context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target);
491 context.setAttribute(ExecutionContext.HTTP_REQUEST, request);
492 context.setAttribute(ExecutionContext.HTTP_RESPONSE, out);
493 context.setAttribute(ExecutionContext.HTTP_REQ_SENT, Boolean.TRUE);
494 }
495 return out;
496 }
497
498 private HttpResponse revalidateCacheEntry(final HttpHost target,
499 final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry,
500 final Date now) throws ClientProtocolException {
501
502 try {
503 if (asynchRevalidator != null
504 && !staleResponseNotAllowed(request, entry, now)
505 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
506 log.trace("Serving stale with asynchronous revalidation");
507 final HttpResponse resp = generateCachedResponse(request, context, entry, now);
508
509 asynchRevalidator.revalidateCacheEntry(target, request, context, entry);
510
511 return resp;
512 }
513 return revalidateCacheEntry(target, request, context, entry);
514 } catch (final IOException ioex) {
515 return handleRevalidationFailure(request, context, entry, now);
516 } catch (final ProtocolException e) {
517 throw new ClientProtocolException(e);
518 }
519 }
520
521 private HttpResponse handleCacheMiss(final HttpHost target, final HttpRequestWrapper request,
522 final HttpContext context) throws IOException {
523 recordCacheMiss(target, request);
524
525 if (!mayCallBackend(request)) {
526 return new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
527 "Gateway Timeout");
528 }
529
530 final Map<String, Variant> variants =
531 getExistingCacheVariants(target, request);
532 if (variants != null && variants.size() > 0) {
533 return negotiateResponseFromVariants(target, request, context, variants);
534 }
535
536 return callBackend(target, request, context);
537 }
538
539 private HttpCacheEntry satisfyFromCache(final HttpHost target, final HttpRequestWrapper request) {
540 HttpCacheEntry entry = null;
541 try {
542 entry = responseCache.getCacheEntry(target, request);
543 } catch (final IOException ioe) {
544 log.warn("Unable to retrieve entries from cache", ioe);
545 }
546 return entry;
547 }
548
549 private HttpResponse getFatallyNoncompliantResponse(final HttpRequestWrapper request,
550 final HttpContext context) {
551 HttpResponse fatalErrorResponse = null;
552 final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
553
554 for (final RequestProtocolError error : fatalError) {
555 setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
556 fatalErrorResponse = requestCompliance.getErrorForRequest(error);
557 }
558 return fatalErrorResponse;
559 }
560
561 private Map<String, Variant> getExistingCacheVariants(final HttpHost target,
562 final HttpRequestWrapper request) {
563 Map<String,Variant> variants = null;
564 try {
565 variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
566 } catch (final IOException ioe) {
567 log.warn("Unable to retrieve variant entries from cache", ioe);
568 }
569 return variants;
570 }
571
572 private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
573 cacheMisses.getAndIncrement();
574 if (log.isTraceEnabled()) {
575 final RequestLine rl = request.getRequestLine();
576 log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
577 }
578 }
579
580 private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
581 cacheHits.getAndIncrement();
582 if (log.isTraceEnabled()) {
583 final RequestLine rl = request.getRequestLine();
584 log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
585 }
586 }
587
588 private void recordCacheUpdate(final HttpContext context) {
589 cacheUpdates.getAndIncrement();
590 setResponseStatus(context, CacheResponseStatus.VALIDATED);
591 }
592
593 private void flushEntriesInvalidatedByRequest(final HttpHost target,
594 final HttpRequestWrapper request) {
595 try {
596 responseCache.flushInvalidatedCacheEntriesFor(target, request);
597 } catch (final IOException ioe) {
598 log.warn("Unable to flush invalidated entries from cache", ioe);
599 }
600 }
601
602 private HttpResponse generateCachedResponse(final HttpRequestWrapper request,
603 final HttpContext context, final HttpCacheEntry entry, final Date now) {
604 final HttpResponse cachedResponse;
605 if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
606 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
607 cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
608 } else {
609 cachedResponse = responseGenerator.generateResponse(request, entry);
610 }
611 setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
612 if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
613 cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
614 }
615 return cachedResponse;
616 }
617
618 private HttpResponse handleRevalidationFailure(final HttpRequestWrapper request,
619 final HttpContext context, final HttpCacheEntry entry, final Date now) {
620 if (staleResponseNotAllowed(request, entry, now)) {
621 return generateGatewayTimeout(context);
622 } else {
623 return unvalidatedCacheHit(request, context, entry);
624 }
625 }
626
627 private HttpResponse generateGatewayTimeout(final HttpContext context) {
628 setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
629 return new BasicHttpResponse(HttpVersion.HTTP_1_1,
630 HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout");
631 }
632
633 private HttpResponse unvalidatedCacheHit(
634 final HttpRequestWrapper request,
635 final HttpContext context,
636 final HttpCacheEntry entry) {
637 final HttpResponse cachedResponse = responseGenerator.generateResponse(request, entry);
638 setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
639 cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
640 return cachedResponse;
641 }
642
643 private boolean staleResponseNotAllowed(final HttpRequestWrapper request,
644 final HttpCacheEntry entry, final Date now) {
645 return validityPolicy.mustRevalidate(entry)
646 || (isSharedCache() && validityPolicy.proxyRevalidate(entry))
647 || explicitFreshnessRequest(request, entry, now);
648 }
649
650 private boolean mayCallBackend(final HttpRequestWrapper request) {
651 for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
652 for (final HeaderElement elt : h.getElements()) {
653 if ("only-if-cached".equals(elt.getName())) {
654 log.trace("Request marked only-if-cached");
655 return false;
656 }
657 }
658 }
659 return true;
660 }
661
662 private boolean explicitFreshnessRequest(final HttpRequestWrapper request, final HttpCacheEntry entry, final Date now) {
663 for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
664 for(final HeaderElement elt : h.getElements()) {
665 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
666 try {
667 final int maxstale = Integer.parseInt(elt.getValue());
668 final long age = validityPolicy.getCurrentAgeSecs(entry, now);
669 final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
670 if (age - lifetime > maxstale) {
671 return true;
672 }
673 } catch (final NumberFormatException nfe) {
674 return true;
675 }
676 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
677 || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
678 return true;
679 }
680 }
681 }
682 return false;
683 }
684
685 private String generateViaHeader(final HttpMessage msg) {
686
687 final ProtocolVersion pv = msg.getProtocolVersion();
688 final String existingEntry = viaHeaders.get(pv);
689 if (existingEntry != null) {
690 return existingEntry;
691 }
692
693 final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
694 final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
695
696 final String value;
697 if ("http".equalsIgnoreCase(pv.getProtocol())) {
698 value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getMajor(), pv.getMinor(),
699 release);
700 } else {
701 value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), pv.getMajor(),
702 pv.getMinor(), release);
703 }
704 viaHeaders.put(pv, value);
705
706 return value;
707 }
708
709 private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
710 if (context != null) {
711 context.setAttribute(CACHE_RESPONSE_STATUS, value);
712 }
713 }
714
715
716
717
718
719
720
721 public boolean supportsRangeAndContentRangeHeaders() {
722 return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
723 }
724
725
726
727
728
729
730
731
732 public boolean isSharedCache() {
733 return sharedCache;
734 }
735
736 Date getCurrentDate() {
737 return new Date();
738 }
739
740 boolean clientRequestsOurOptions(final HttpRequest request) {
741 final RequestLine line = request.getRequestLine();
742
743 if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
744 return false;
745 }
746
747 if (!"*".equals(line.getUri())) {
748 return false;
749 }
750
751 final Header h = request.getFirstHeader(HeaderConstants.MAX_FORWARDS);
752 if (!"0".equals(h != null ? h.getValue() : null)) {
753 return false;
754 }
755
756 return true;
757 }
758
759 HttpResponse callBackend(final HttpHost target, final HttpRequestWrapper request, final HttpContext context)
760 throws IOException {
761
762 final Date requestDate = getCurrentDate();
763
764 log.trace("Calling the backend");
765 final HttpResponse backendResponse = backend.execute(target, request, context);
766 backendResponse.addHeader("Via", generateViaHeader(backendResponse));
767 return handleBackendResponse(target, request, requestDate, getCurrentDate(),
768 backendResponse);
769
770 }
771
772 private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
773 final HttpCacheEntry cacheEntry) {
774 final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
775 final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
776 if (entryDateHeader != null && responseDateHeader != null) {
777 try {
778 final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
779 final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
780 if (respDate.before(entryDate)) {
781 return true;
782 }
783 } catch (final DateParseException e) {
784
785
786
787
788 }
789 }
790 return false;
791 }
792
793 HttpResponse negotiateResponseFromVariants(final HttpHost target,
794 final HttpRequestWrapper request, final HttpContext context,
795 final Map<String, Variant> variants) throws IOException {
796 final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
797 .buildConditionalRequestFromVariants(request, variants);
798
799 final Date requestDate = getCurrentDate();
800 final HttpResponse backendResponse = backend.execute(target, conditionalRequest, context);
801 final Date responseDate = getCurrentDate();
802
803 backendResponse.addHeader("Via", generateViaHeader(backendResponse));
804
805 if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
806 return handleBackendResponse(target, request, requestDate, responseDate, backendResponse);
807 }
808
809 final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
810 if (resultEtagHeader == null) {
811 log.warn("304 response did not contain ETag");
812 return callBackend(target, request, context);
813 }
814
815 final String resultEtag = resultEtagHeader.getValue();
816 final Variant matchingVariant = variants.get(resultEtag);
817 if (matchingVariant == null) {
818 log.debug("304 response did not contain ETag matching one sent in If-None-Match");
819 return callBackend(target, request, context);
820 }
821
822 final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
823
824 if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
825 IOUtils.consume(backendResponse.getEntity());
826 return retryRequestUnconditionally(target, request, context,
827 matchedEntry);
828 }
829
830 recordCacheUpdate(context);
831
832 final HttpCacheEntry responseEntry = getUpdatedVariantEntry(target,
833 conditionalRequest, requestDate, responseDate, backendResponse,
834 matchingVariant, matchedEntry);
835
836 final HttpResponse resp = responseGenerator.generateResponse(request, responseEntry);
837 tryToUpdateVariantMap(target, request, matchingVariant);
838
839 if (shouldSendNotModifiedResponse(request, responseEntry)) {
840 return responseGenerator.generateNotModifiedResponse(responseEntry);
841 }
842
843 return resp;
844 }
845
846 private HttpResponse retryRequestUnconditionally(final HttpHost target,
847 final HttpRequestWrapper request, final HttpContext context,
848 final HttpCacheEntry matchedEntry) throws IOException {
849 final HttpRequestWrapper unconditional = conditionalRequestBuilder
850 .buildUnconditionalRequest(request, matchedEntry);
851 return callBackend(target, unconditional, context);
852 }
853
854 private HttpCacheEntry getUpdatedVariantEntry(final HttpHost target,
855 final HttpRequestWrapper conditionalRequest, final Date requestDate,
856 final Date responseDate, final HttpResponse backendResponse,
857 final Variant matchingVariant, final HttpCacheEntry matchedEntry) {
858 HttpCacheEntry responseEntry = matchedEntry;
859 try {
860 responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
861 matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
862 } catch (final IOException ioe) {
863 log.warn("Could not update cache entry", ioe);
864 }
865 return responseEntry;
866 }
867
868 private void tryToUpdateVariantMap(final HttpHost target, final HttpRequestWrapper request,
869 final Variant matchingVariant) {
870 try {
871 responseCache.reuseVariantEntryFor(target, request, matchingVariant);
872 } catch (final IOException ioe) {
873 log.warn("Could not update cache entry to reuse variant", ioe);
874 }
875 }
876
877 private boolean shouldSendNotModifiedResponse(final HttpRequestWrapper request,
878 final HttpCacheEntry responseEntry) {
879 return (suitabilityChecker.isConditional(request)
880 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
881 }
882
883 HttpResponse revalidateCacheEntry(
884 final HttpHost target,
885 final HttpRequestWrapper request,
886 final HttpContext context,
887 final HttpCacheEntry cacheEntry) throws IOException, ProtocolException {
888
889 final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
890
891 Date requestDate = getCurrentDate();
892 HttpResponse backendResponse = backend.execute(target, conditionalRequest, context);
893 Date responseDate = getCurrentDate();
894
895 if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
896 IOUtils.consume(backendResponse.getEntity());
897 final HttpRequest unconditional = conditionalRequestBuilder
898 .buildUnconditionalRequest(request, cacheEntry);
899 requestDate = getCurrentDate();
900 backendResponse = backend.execute(target, unconditional, context);
901 responseDate = getCurrentDate();
902 }
903
904 backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
905
906 final int statusCode = backendResponse.getStatusLine().getStatusCode();
907 if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
908 recordCacheUpdate(context);
909 }
910
911 if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
912 final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(target, request, cacheEntry,
913 backendResponse, requestDate, responseDate);
914 if (suitabilityChecker.isConditional(request)
915 && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
916 return responseGenerator.generateNotModifiedResponse(updatedEntry);
917 }
918 return responseGenerator.generateResponse(request, updatedEntry);
919 }
920
921 if (staleIfErrorAppliesTo(statusCode)
922 && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
923 && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
924 final HttpResponse cachedResponse = responseGenerator.generateResponse(request, cacheEntry);
925 cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
926 final HttpEntity errorBody = backendResponse.getEntity();
927 if (errorBody != null) {
928 IOUtils.consume(errorBody);
929 }
930 return cachedResponse;
931 }
932
933 return handleBackendResponse(target, conditionalRequest, requestDate, responseDate,
934 backendResponse);
935 }
936
937 private boolean staleIfErrorAppliesTo(final int statusCode) {
938 return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
939 || statusCode == HttpStatus.SC_BAD_GATEWAY
940 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
941 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
942 }
943
944 HttpResponse handleBackendResponse(
945 final HttpHost target,
946 final HttpRequestWrapper request,
947 final Date requestDate,
948 final Date responseDate,
949 final HttpResponse backendResponse) throws IOException {
950
951 log.trace("Handling Backend response");
952 responseCompliance.ensureProtocolCompliance(request, backendResponse);
953
954 final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
955 responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
956 if (cacheable &&
957 !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
958 try {
959 storeRequestIfModifiedSinceFor304Response(request, backendResponse);
960 return responseCache.cacheAndReturnResponse(target, request, backendResponse, requestDate,
961 responseDate);
962 } catch (final IOException ioe) {
963 log.warn("Unable to store entries in cache", ioe);
964 }
965 }
966 if (!cacheable) {
967 try {
968 responseCache.flushCacheEntriesFor(target, request);
969 } catch (final IOException ioe) {
970 log.warn("Unable to flush invalid cache entries", ioe);
971 }
972 }
973 return backendResponse;
974 }
975
976
977
978
979
980
981
982
983
984 private void storeRequestIfModifiedSinceFor304Response(
985 final HttpRequest request, final HttpResponse backendResponse) {
986 if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
987 final Header h = request.getFirstHeader("If-Modified-Since");
988 if (h != null) {
989 backendResponse.addHeader("Last-Modified", h.getValue());
990 }
991 }
992 }
993
994 private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequest request,
995 final HttpResponse backendResponse) {
996 HttpCacheEntry existing = null;
997 try {
998 existing = responseCache.getCacheEntry(target, request);
999 } catch (final IOException ioe) {
1000
1001 }
1002 if (existing == null) {
1003 return false;
1004 }
1005 final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
1006 if (entryDateHeader == null) {
1007 return false;
1008 }
1009 final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
1010 if (responseDateHeader == null) {
1011 return false;
1012 }
1013 try {
1014 final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
1015 final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
1016 return responseDate.before(entryDate);
1017 } catch (final DateParseException e) {
1018
1019 }
1020 return false;
1021 }
1022
1023 static class AsynchronousValidator {
1024 private final CachingHttpClient cachingClient;
1025 private final ExecutorService executor;
1026 private final Set<String> queued;
1027 private final CacheKeyGenerator cacheKeyGenerator;
1028
1029 private final Log log = LogFactory.getLog(getClass());
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043 public AsynchronousValidator(final CachingHttpClient cachingClient,
1044 final CacheConfig config) {
1045 this(cachingClient,
1046 new ThreadPoolExecutor(config.getAsynchronousWorkersCore(),
1047 config.getAsynchronousWorkersMax(),
1048 config.getAsynchronousWorkerIdleLifetimeSecs(),
1049 TimeUnit.SECONDS,
1050 new ArrayBlockingQueue<Runnable>(config.getRevalidationQueueSize()))
1051 );
1052 }
1053
1054
1055
1056
1057
1058
1059
1060
1061 AsynchronousValidator(final CachingHttpClient cachingClient,
1062 final ExecutorService executor) {
1063 this.cachingClient = cachingClient;
1064 this.executor = executor;
1065 this.queued = new HashSet<String>();
1066 this.cacheKeyGenerator = new CacheKeyGenerator();
1067 }
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077 public synchronized void revalidateCacheEntry(final HttpHost target,
1078 final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry) {
1079
1080 final String uri = cacheKeyGenerator.getVariantURI(target, request, entry);
1081
1082 if (!queued.contains(uri)) {
1083 final AsynchronousValidationRequest revalidationRequest =
1084 new AsynchronousValidationRequest(this, cachingClient, target,
1085 request, context, entry, uri);
1086
1087 try {
1088 executor.execute(revalidationRequest);
1089 queued.add(uri);
1090 } catch (final RejectedExecutionException ree) {
1091 log.debug("Revalidation for [" + uri + "] not scheduled: " + ree);
1092 }
1093 }
1094 }
1095
1096
1097
1098
1099
1100
1101
1102
1103 synchronized void markComplete(final String identifier) {
1104 queued.remove(identifier);
1105 }
1106
1107 Set<String> getScheduledIdentifiers() {
1108 return Collections.unmodifiableSet(queued);
1109 }
1110
1111 ExecutorService getExecutor() {
1112 return executor;
1113 }
1114 }
1115
1116 static class AsynchronousValidationRequest implements Runnable {
1117 private final AsynchronousValidator parent;
1118 private final CachingHttpClient cachingClient;
1119 private final HttpHost target;
1120 private final HttpRequestWrapper request;
1121 private final HttpContext context;
1122 private final HttpCacheEntry cacheEntry;
1123 private final String identifier;
1124
1125 private final Log log = LogFactory.getLog(getClass());
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138 AsynchronousValidationRequest(final AsynchronousValidator parent,
1139 final CachingHttpClient cachingClient, final HttpHost target,
1140 final HttpRequestWrapper request, final HttpContext context,
1141 final HttpCacheEntry cacheEntry,
1142 final String identifier) {
1143 this.parent = parent;
1144 this.cachingClient = cachingClient;
1145 this.target = target;
1146 this.request = request;
1147 this.context = context;
1148 this.cacheEntry = cacheEntry;
1149 this.identifier = identifier;
1150 }
1151
1152 @Override
1153 public void run() {
1154 try {
1155 cachingClient.revalidateCacheEntry(target, request, context, cacheEntry);
1156 } catch (final IOException ioe) {
1157 log.debug("Asynchronous revalidation failed due to exception: " + ioe);
1158 } catch (final ProtocolException pe) {
1159 log.error("ProtocolException thrown during asynchronous revalidation: " + pe);
1160 } finally {
1161 parent.markComplete(identifier);
1162 }
1163 }
1164
1165 String getIdentifier() {
1166 return identifier;
1167 }
1168
1169 }
1170
1171 }