001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.juneau.rest.client.okhttp;
018
019import static org.apache.juneau.commons.utils.Shorts.*;
020
021import java.io.*;
022import java.net.*;
023import java.util.*;
024import java.util.concurrent.*;
025import java.util.regex.*;
026
027import org.apache.juneau.http.remote.*;
028import org.apache.juneau.rest.client.*;
029
030import okhttp3.*;
031import okio.*;
032
033/**
034 * {@link HttpTransport} implementation backed by OkHttp 5.x.
035 *
036 * <p>
037 * This transport is auto-discovered via {@link java.util.ServiceLoader} when
038 * {@code com.squareup.okhttp3:okhttp-jvm} is on the classpath.  You can also instantiate it explicitly:
039 *
040 * <p class='bjava'>
041 *    <jv>transport</jv> = OkHttpTransport.<jsm>builder</jsm>()
042 *       .httpClient(<jk>new</jk> OkHttpClient())
043 *       .build();
044 *
045 *    <jv>client</jv> = RestClient.<jsm>builder</jsm>()
046 *       .transport(<jv>transport</jv>)
047 *       .build();
048 * </p>
049 *
050 * <p>
051 * OkHttp's {@link RequestBody#writeTo(BufferedSink)} streams the request body directly to the socket
052 * without in-memory buffering for bodies with a known content length.
053 *
054 * <h5 class='section'>See Also:</h5><ul>
055 *    <li class='link'><a class="doclink" href="https://juneau.apache.org/docs/topics/NextGenRestClient">Next-Generation REST Client</a>
056 * </ul>
057 *
058 * @since 9.2.1
059 */
060@SuppressWarnings({
061   "resource" // Transport methods return Closeables owned by the caller; Eclipse JDT @Owning warning is by design.
062})
063public final class OkHttpTransport implements HttpTransport {
064
065   // OkHttp's own IP-literal test (okhttp3.internal HostnamesCommon): such hosts are connected to without a Dns lookup.
066   private static final Pattern OKHTTP_IP_LITERAL = Pattern.compile("([0-9a-fA-F]*:[0-9a-fA-F:.]*)|([\\d.]+)");
067
068   private final OkHttpClient httpClient;
069   private final OkHttpClient policyHttpClient;
070   // One further pin-on-connect client per distinct non-empty per-client allow-list, created on first use.  Each has its
071   // own connection pool, so a connection opened under one allow-list is never reused by a request with another.
072   private final Map<RemoteUrlPolicy.AllowedRanges,OkHttpClient> rangedPolicyHttpClients = new ConcurrentHashMap<>();
073
074   OkHttpTransport(OkHttpTransportBuilder builder) {
075      // A caller-supplied client gets the redirect credential guard on a derived client: newBuilder() shares its
076      // connection pool, dispatcher and every other setting, and the caller's own client is left unchanged.
077      this.httpClient = builder.httpClient != null ? withCredentialGuard(builder.httpClient) : createDefaultHttpClient();
078      // A dedicated client -- sharing this client's connection pool/dispatcher via newBuilder(), but with its own
079      // pin-on-connect Dns and with auto-redirect-following disabled -- used only for SSRF-guard-active @Remote
080      // calls (see executeWithPolicy); ordinary (non-@Remote) calls on httpClient are completely unaffected.
081      // Unlike the shared-connection-manager engines (Apache HttpClient, Jetty), OkHttp's Dns/followRedirects are
082      // per-client (not tied to the shared connection pool), so this derivation works even for a caller-supplied
083      // httpClient.  The exception is a proxy: OkHttp asks Dns for the proxy's address, not the target's (the proxy
084      // resolves the target), so the pin cannot apply -- see supportsUrlPolicy() and requireNoProxy().
085      this.policyHttpClient = this.httpClient.newBuilder()
086         .followRedirects(false)
087         .dns(PolicyPinningDns.INSTANCE)
088         .build();
089   }
090
091   private static OkHttpClient createDefaultHttpClient() {
092      return withCredentialGuard(new OkHttpClient());
093   }
094
095   // OkHttp's own redirect follower keeps a caller-set Cookie, so every client this transport sends through has the
096   // guard as a network interceptor, which runs for each hop.  The first-hop recorder is the last application
097   // interceptor, so the origin is the URL after any caller application interceptor has rewritten it.
098   private static OkHttpClient withCredentialGuard(OkHttpClient client) {
099      return client.newBuilder()
100         .addInterceptor(OkHttpRedirectCredentialGuard.FIRST_HOP_RECORDER)
101         .addNetworkInterceptor(new OkHttpRedirectCredentialGuard())
102         .build();
103   }
104
105   /**
106    * Returns a new builder for this transport.
107    *
108    * @return A new builder. Never <jk>null</jk>.
109    */
110   public static OkHttpTransportBuilder builder() {
111      return new OkHttpTransportBuilder();
112   }
113
114   /**
115    * Returns a new instance backed by a default {@link OkHttpClient}.
116    *
117    * @return A new instance. Never <jk>null</jk>.
118    */
119   public static OkHttpTransport create() {
120      return builder().build();
121   }
122
123   @Override /* HttpTransport */
124   public TransportResponse execute(TransportRequest request) throws TransportException {
125      if (request.isSsrfGuardActive())
126         return PolicyEnforcedRedirects.execute(request, this::sendOncePinned);
127      return sendOnce(request, httpClient);
128   }
129
130   /**
131    * {@inheritDoc}
132    *
133    * <p>
134    * Reports <jk>false</jk> when the {@link OkHttpClient} has an explicit {@linkplain OkHttpClient#proxy() proxy}: a
135    * proxy resolves the target host itself, so the pin-on-connect {@link okhttp3.Dns} is never consulted for the target.
136    * A client that selects a proxy per URL through a {@link ProxySelector} is accepted here, but a guarded request
137    * whose URL is routed through a proxy is refused when sent (fail closed).
138    */
139   @Override /* HttpTransport */
140   public boolean supportsUrlPolicy() {
141      var proxy = httpClient.proxy();
142      return proxy == null || proxy.type() == Proxy.Type.DIRECT;
143   }
144
145   // Performs one pin-on-connect hop via the dedicated, redirect-disabled policyHttpClient.
146   private TransportResponse sendOncePinned(TransportRequest request) throws TransportException {
147      try {
148         var url = HttpUrl.get(request.getUri().toString());
149         requireNoProxy(url);
150         requireAllowedIpLiteral(url.host(), request.getAllowedAddressRanges());
151      } catch (IllegalArgumentException e) {
152         throw new TransportException("HTTP transport error: " + e.getMessage(), e);
153      }
154      return sendOnce(request, policyClientFor(request.getAllowedAddressRanges()));
155   }
156
157   private OkHttpClient policyClientFor(RemoteUrlPolicy.AllowedRanges ranges) {
158      if (ranges.isEmpty())
159         return policyHttpClient;
160      return rangedPolicyHttpClients.computeIfAbsent(ranges, r -> policyHttpClient.newBuilder()
161         .dns(new PolicyPinningDns(r))
162         .connectionPool(new ConnectionPool())
163         .build());
164   }
165
166   // A proxy (SOCKS, or HTTP/CONNECT) resolves the target host itself, so OkHttp never asks Dns about the target and the
167   // pin cannot apply.  Fail closed rather than send a guarded request through one.
168   private void requireNoProxy(HttpUrl url) {
169      var explicit = httpClient.proxy();
170      var uri = url.uri();
171      // Like OkHttp, skip the ProxySelector when java.net.URI cannot parse a host (e.g. an underscore host name).
172      List<Proxy> proxies;
173      if (explicit != null)
174         proxies = List.of(explicit);
175      else if (uri.getHost() == null)
176         proxies = List.of(Proxy.NO_PROXY);
177      else
178         proxies = selectedProxies(uri);
179      for (var p : proxies)
180         if (p.type() != Proxy.Type.DIRECT)
181            throw iaex("Request to '%s' would be routed through a proxy, which resolves the target host itself and bypasses the SSRF guard's address pinning; use a client without a proxy (or set allowPrivateUrls)", url.host());
182   }
183
184   // Like OkHttp, a ProxySelector answer of null or empty means no proxy.
185   private List<Proxy> selectedProxies(URI uri) {
186      var selected = httpClient.proxySelector().select(uri);
187      return selected == null || selected.isEmpty() ? List.of(Proxy.NO_PROXY) : selected;
188   }
189
190   // OkHttp connects to any host it considers an IP literal without consulting Dns, so the pin never sees one; apply the
191   // same address check here.  Its literal test is deliberately broader than dotted-quad: an all-digit/dot host such as
192   // 2130706433 or 127.1 is parsed by InetAddress to 127.0.0.1.  (Host names are checked by PolicyPinningDns.)
193   // The literal is parsed locally and never resolved: a host that matches the pattern but is not a valid literal
194   // (1.2.3.4.5, 127.0.0.1.) would otherwise fall through to a DNS lookup here and a second one in OkHttp, so it is refused.
195   private static void requireAllowedIpLiteral(String host, RemoteUrlPolicy.AllowedRanges ranges) {
196      if (! OKHTTP_IP_LITERAL.matcher(host).matches())
197         return;
198      var address = parseIpLiteral(host);
199      if (address == null)
200         throw iaex("Host '%s' matches the IP-literal form but is not a valid IP literal; refusing to resolve it", host);
201      if (RemoteUrlPolicy.isDeniedAddress(address, ranges))
202         throw iaex("No allowed address for host '%s': the IP literal is private/loopback/link-local/reserved (set allowPrivateUrls to allow local-dev targets)", host);
203   }
204
205   // Returns the address for an IP-literal host without any name lookup, or null if it is not a valid literal.
206   private static InetAddress parseIpLiteral(String host) {
207      try {
208         if (host.indexOf(':') != -1)
209            return InetAddress.getByName("[" + host + "]"); // brackets make the JDK treat it as IPv6 only, never DNS
210         var parts = host.split("\\.", -1);
211         if (parts.length > 4)
212            return null;
213         var value = 0L;
214         for (var i = 0; i < parts.length; i++) {
215            var part = parts[i];
216            if (part.isEmpty() || part.length() > 10 || (part.length() > 1 && part.charAt(0) == '0'))
217               return null;
218            var n = Long.parseLong(part);
219            var last = i == parts.length - 1;
220            if (n > (last ? (1L << (8 * (5 - parts.length))) - 1 : 255))
221               return null;
222            value |= last ? n : n << (8 * (3 - i));
223         }
224         return InetAddress.getByAddress(new byte[]{(byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte)value});
225      } catch (UnknownHostException e) {
226         return null;
227      }
228   }
229
230   private static TransportResponse sendOnce(TransportRequest request, OkHttpClient client) throws TransportException {
231      var okRequest = buildOkRequest(request);
232      var timeout = request.getTimeout();
233      // Per-request read timeout (response-header wait and gaps between body reads); newBuilder() shares the connection pool.
234      var effective = timeout == null ? client : client.newBuilder().readTimeout(timeout).build();
235      Response okResponse;
236      try {
237         okResponse = effective.newCall(okRequest).execute();
238      } catch (IOException | IllegalArgumentException e) {
239         // IllegalArgumentException: any IAE raised while executing the call, on both the guarded and the ordinary
240         // path, is surfaced as a TransportException.  The expected source is a Dns.lookup failure, which OkHttp
241         // propagates unchanged -- notably PolicyPinningDns refusing a host whose every resolved address is denied.
242         throw new TransportException("HTTP transport error: " + e.getMessage(), e);
243      }
244      try {
245         return buildTransportResponse(okResponse);
246      } catch (TransportException | RuntimeException e) {
247         // If response-body wiring fails the closeCallback never runs, so close here to avoid leaking the connection.
248         okResponse.close();
249         throw e;
250      }
251   }
252
253   @Override /* Closeable */
254   public void close() {
255      httpClient.dispatcher().executorService().shutdown();
256      httpClient.connectionPool().evictAll();
257      for (var c : rangedPolicyHttpClients.values())
258         c.connectionPool().evictAll();
259   }
260
261   // -----------------------------------------------------------------------------------------------------------------
262   // Internal helpers
263   // -----------------------------------------------------------------------------------------------------------------
264
265   private static Request buildOkRequest(TransportRequest request) {
266      var builder = new Request.Builder().url(request.getUri().toString());
267      for (var h : request.getHeaders())
268         builder.addHeader(h.name(), h.value());
269      builder.method(request.getMethod(), buildOkBody(request.getBody(), request.getMethod()));
270      return builder.build();
271   }
272
273   private static RequestBody buildOkBody(TransportBody body, String method) {
274      if (body == null) {
275         // OkHttp requires a non-null body for POST/PUT/PATCH even if empty
276         return requiresBody(method) ? RequestBody.create(new byte[0], null) : null;
277      }
278      var ct = body.getContentType();
279      var mediaType = ct != null ? MediaType.parse(ct) : null;
280      return new RequestBody() {
281         @Override
282         public MediaType contentType() {
283            return mediaType;
284         }
285
286         @Override
287         public long contentLength() {
288            return body.getContentLength();
289         }
290
291         @Override
292         public void writeTo(BufferedSink sink) throws IOException {
293            body.writeTo(sink.outputStream());
294         }
295      };
296   }
297
298   private static boolean requiresBody(String method) {
299      return eqic("POST", method) || eqic("PUT", method) || eqic("PATCH", method);
300   }
301
302   @SuppressWarnings({
303      "java:S2095" // okResponse is closed via TransportResponse.closeCallback, not here
304   })
305   private static TransportResponse buildTransportResponse(Response okResponse) throws TransportException {
306      var builder = TransportResponse.builder()
307         .statusCode(okResponse.code())
308         .reasonPhrase(okResponse.message())
309         .closeCallback(okResponse);
310      for (var i = 0; i < okResponse.headers().size(); i++)
311         builder.header(okResponse.headers().name(i), okResponse.headers().value(i));
312      try {
313         builder.body(okResponse.body().byteStream()); // body() is @NonNull in OkHttp 5.x
314      } catch (Exception e) {
315         throw new TransportException("Failed to read response body: " + e.getMessage(), e);
316      }
317      return builder.build();
318   }
319}