001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.juneau.rest.server.convention; 018 019import static org.apache.juneau.commons.utils.Shorts.*; 020 021import java.io.*; 022import java.time.*; 023import java.time.format.*; 024import java.util.*; 025 026import org.apache.juneau.rest.server.*; 027import org.apache.juneau.rest.server.servlet.*; 028 029/** 030 * Mixin that serves SEO-conventional endpoints {@code /robots.txt} and {@code /sitemap.xml}. 031 * 032 * <p> 033 * Sibling of {@link FaviconMixin} ({@code /favicon.ico}), 034 * {@link VersionMixin} ({@code /version}), and 035 * {@link WellKnownMixin} ({@code /.well-known/*}). All four classes live in the 036 * {@code org.apache.juneau.rest.server.convention} convention-endpoints mixin pack. 037 * 038 * <p> 039 * Compose into a host resource via {@link Rest#mixins() @Rest(mixins=SeoMixin.class)}; 040 * the {@code /robots.txt} and {@code /sitemap.xml} URLs become available alongside the host's own 041 * endpoints with no further wiring. 042 * 043 * <h5 class='section'>Hardcoded mount paths:</h5> 044 * 045 * <p> 046 * Unlike the sibling api-docs and ops mixins (see {@link org.apache.juneau.rest.server.docs.SwaggerMixin}, 047 * {@link VersionMixin}, etc.), the two mount paths here are <b>not</b> SVL-configurable. 048 * {@code /robots.txt} is fixed by the Robots Exclusion Protocol 049 * (<a href="https://www.rfc-editor.org/rfc/rfc9309">RFC 9309</a>) which prescribes that 050 * crawlers fetch the policy from {@code /robots.txt} at the site root; {@code /sitemap.xml} is 051 * fixed by the <a href="https://www.sitemaps.org/protocol.html">sitemaps.org protocol</a> 052 * (and consumed by search engines configured to look for the file at that exact path). A 053 * runtime mount-path override here would have no practical effect. 054 * 055 * <h5 class='section'>Mixin-only deployment:</h5> 056 * 057 * <p> 058 * This resource is designed for composition via {@code @Rest(mixins=...)}. The two mount paths 059 * ({@code /robots.txt}, {@code /sitemap.xml}) are pinned at the op level by 060 * {@link RestGet @RestGet(path=...)} on the handler methods; a class-level 061 * {@code @Rest(paths=...)} declaration would be silently ignored under the mixin pattern (see 062 * {@link Rest#paths() @Rest(paths)} Javadoc). 063 * 064 * <h5 class='figure'>Composition example:</h5> 065 * 066 * <p class='bjava'> 067 * <ja>@Rest</ja>(path=<js>"/api"</js>, mixins=SeoMixin.<jk>class</jk>) 068 * <jk>public class</jk> ApiResource <jk>extends</jk> RestServlet { } 069 * 070 * <jc>// Override defaults via a @Bean factory:</jc> 071 * <ja>@Bean</ja> SeoMixin seo() { 072 * <jk>return</jk> SeoMixin.<jsm>create</jsm>() 073 * .robotsAllow(<js>"*"</js>, <js>"/"</js>) 074 * .sitemapEntry(<js>"/api/items"</js>) 075 * .build(); 076 * } 077 * </p> 078 * 079 * <h5 class='section'>Defaults & behavior:</h5> 080 * 081 * <ul class='spaced-list'> 082 * <li><b>Robots policy — deny-all by default.</b> {@code GET /robots.txt} returns 083 * {@code "User-agent: *\nDisallow: /\n"} unless the builder specifies a different policy via 084 * {@link Builder#robotsAllow(String,String...) robotsAllow(...)} or 085 * {@link Builder#robotsDisallow(String,String...) robotsDisallow(...)}. The deny-all default 086 * is intentional — mounting the mixin without thinking should not auto-opt the service 087 * into search-engine indexing. 088 * <li><b>Sitemap empty by default.</b> {@code GET /sitemap.xml} returns an empty 089 * {@code <urlset>} when no entries are configured. Add entries via 090 * {@link Builder#sitemapEntry(String) sitemapEntry(String)} or 091 * {@link Builder#sitemapEntry(String,ZonedDateTime,String,Double) sitemapEntry(String,ZonedDateTime,String,Double)}. 092 * <li><b>Content types.</b> {@code /robots.txt} returns {@code text/plain; charset=UTF-8}; 093 * {@code /sitemap.xml} returns {@code application/xml; charset=UTF-8}. 094 * <li>Both endpoints are excluded from generated Swagger / OpenAPI specs via 095 * {@link OpSwagger#ignore() @OpSwagger(ignore=true)}. 096 * </ul> 097 * 098 * <h5 class='section'>See Also:</h5><ul> 099 * <li class='jc'>{@link FaviconMixin} 100 * <li class='jc'>{@link VersionMixin} 101 * <li class='jc'>{@link WellKnownMixin} 102 * <li class='link'><a class="doclink" href="https://www.rfc-editor.org/rfc/rfc9309">RFC 9309 — Robots Exclusion Protocol</a> 103 * <li class='link'><a class="doclink" href="https://www.sitemaps.org/protocol.html">sitemaps.org — Sitemap protocol</a> 104 * <li class='link'><a class="doclink" href="https://juneau.apache.org/docs/topics/RestServerComposition">REST Server — Mixins and Multi-Mount Paths</a> 105 * </ul> 106 * 107 * @since 10.0.0 108 */ 109// @formatter:off 110@Rest 111public class SeoMixin extends RestMixin { 112 113 /** Default robots policy: deny everything. */ 114 public static final String DEFAULT_ROBOTS = "User-agent: *\nDisallow: /\n"; 115 116 /** 117 * Creates a new builder. 118 * 119 * @return A new builder. 120 */ 121 public static Builder create() { 122 return new Builder(); 123 } 124 125 private final String robotsTxt; 126 private final List<SitemapEntry> sitemapEntries; 127 128 /** No-arg constructor used when a host registers the mixin without builder configuration. */ 129 public SeoMixin() { 130 this(create()); 131 } 132 133 /** 134 * Builder constructor. 135 * 136 * <p> 137 * Stashes the programmatic {@link RestBuilder}(the builder itself, carrying any {@code @Rest}-level 138 * overrides such as {@code path}) via {@code super(builder)} so those values take precedence over this 139 * mixin's own {@link Rest @Rest} annotation. 140 * 141 * @param builder The builder. Must not be <jk>null</jk>. 142 */ 143 protected SeoMixin(Builder builder) { 144 super(builder); 145 robotsTxt = builder.buildRobots(); 146 sitemapEntries = List.copyOf(builder.sitemapEntries); 147 } 148 149 /** 150 * [GET /robots.txt] — emit the configured robots policy as {@code text/plain}. 151 * 152 * @param res The current REST response. 153 * @throws IOException If an I/O error occurs while writing the response. 154 */ 155 @RestGet( 156 path="/robots.txt", 157 summary="Robots policy", 158 description="Robots Exclusion Protocol policy file (RFC 9309).", 159 swagger=@OpSwagger(ignore=true) 160 ) 161 public void getRobotsTxt(RestResponse res) throws IOException { 162 try (var w = res.getDirectWriter("text/plain; charset=UTF-8")) { 163 w.write(robotsTxt); 164 } 165 } 166 167 /** 168 * [GET /sitemap.xml] — emit the configured sitemap as {@code application/xml}. 169 * 170 * @param res The current REST response. 171 * @throws IOException If an I/O error occurs while writing the response. 172 */ 173 @RestGet( 174 path="/sitemap.xml", 175 summary="Sitemap", 176 description="XML sitemap of indexable URLs (sitemaps.org protocol).", 177 swagger=@OpSwagger(ignore=true) 178 ) 179 public void getSitemap(RestResponse res) throws IOException { 180 try (var w = res.getDirectWriter("application/xml; charset=UTF-8")) { 181 w.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); 182 w.write("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n"); 183 for (var e : sitemapEntries) 184 e.write(w); 185 w.write("</urlset>\n"); 186 } 187 } 188 189 /** 190 * Single sitemap entry (immutable). Composed of a required {@code loc} URL and optional 191 * {@code lastmod}, {@code changefreq}, and {@code priority} elements per the 192 * <a href="https://www.sitemaps.org/protocol.html">sitemaps.org protocol</a>. 193 */ 194 public static final class SitemapEntry { 195 private final String loc; 196 private final ZonedDateTime lastmod; 197 private final String changefreq; 198 private final Double priority; 199 200 SitemapEntry(String loc, ZonedDateTime lastmod, String changefreq, Double priority) { 201 this.loc = loc; 202 this.lastmod = lastmod; 203 this.changefreq = changefreq; 204 this.priority = priority; 205 } 206 207 void write(Writer w) throws IOException { 208 w.write("\t<url>\n"); 209 w.write("\t\t<loc>" + xmlEscape(loc) + "</loc>\n"); 210 if (lastmod != null) 211 w.write("\t\t<lastmod>" + lastmod.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + "</lastmod>\n"); 212 if (changefreq != null) 213 w.write("\t\t<changefreq>" + xmlEscape(changefreq) + "</changefreq>\n"); 214 if (priority != null) 215 w.write("\t\t<priority>" + priority + "</priority>\n"); 216 w.write("\t</url>\n"); 217 } 218 219 private static String xmlEscape(String s) { 220 return s.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'"); 221 } 222 } 223 224 /** 225 * Builder for {@link SeoMixin} instances. 226 * 227 * <p> 228 * Extends {@link org.apache.juneau.rest.server.servlet.RestMixin.Builder RestMixin.Builder} so the mixin's bespoke robots/sitemap setters chain 229 * with true covariant returns alongside the inherited {@link RestBuilder}surface (e.g. {@code path}, 230 * {@code roleGuard}). 231 */ 232 public static class Builder extends RestMixin.Builder<SeoMixin,Builder> { 233 234 private final List<RobotsRule> robotsRules = new ArrayList<>(); 235 private final List<SitemapEntry> sitemapEntries = new ArrayList<>(); 236 private String customRobotsTxt; 237 238 /** Constructor — package access for {@link SeoMixin#create()}. */ 239 protected Builder() { 240 super(SeoMixin.class); 241 } 242 243 /** 244 * Adds an {@code Allow} rule to the robots policy. 245 * 246 * <p> 247 * Multiple invocations append rules in order. The first call to either 248 * {@link #robotsAllow(String,String...) robotsAllow(...)} or 249 * {@link #robotsDisallow(String,String...) robotsDisallow(...)} replaces the deny-all 250 * default; subsequent calls continue to append. 251 * 252 * @param userAgent The user agent (e.g. {@code "*"} or {@code "Googlebot"}). 253 * Must not be <jk>null</jk> or blank. 254 * @param paths Allowed path prefixes (e.g. {@code "/"}). At least one path is required. 255 * @return This object. 256 */ 257 public Builder robotsAllow(String userAgent, String...paths) { 258 robotsRules.add(new RobotsRule(userAgent, true, paths)); 259 return this; 260 } 261 262 /** 263 * Adds a {@code Disallow} rule to the robots policy. 264 * 265 * @param userAgent The user agent (e.g. {@code "*"}). Must not be <jk>null</jk> or blank. 266 * @param paths Disallowed path prefixes. At least one path is required. 267 * @return This object. 268 */ 269 public Builder robotsDisallow(String userAgent, String...paths) { 270 robotsRules.add(new RobotsRule(userAgent, false, paths)); 271 return this; 272 } 273 274 /** 275 * Sets a fully-formed {@code robots.txt} body, overriding any rule-builder calls. 276 * 277 * <p> 278 * Useful when the desired policy doesn't fit the simple 279 * {@code allow}/{@code disallow}/per-user-agent shape of 280 * {@link #robotsAllow(String,String...) robotsAllow(...)} / 281 * {@link #robotsDisallow(String,String...) robotsDisallow(...)} (e.g. when including 282 * {@code Sitemap:} or {@code Crawl-delay:} directives). 283 * 284 * @param value The full {@code robots.txt} body. Must not be <jk>null</jk>. 285 * @return This object. 286 */ 287 public Builder robotsTxt(String value) { 288 customRobotsTxt = value; 289 return this; 290 } 291 292 /** 293 * Adds a sitemap entry with only the {@code loc} URL. 294 * 295 * @param url The fully-qualified URL. 296 * @return This object. 297 */ 298 public Builder sitemapEntry(String url) { 299 return sitemapEntry(url, null, null, null); 300 } 301 302 /** 303 * Adds a sitemap entry with optional {@code lastmod}, {@code changefreq}, and 304 * {@code priority} components. 305 * 306 * @param url The fully-qualified URL. 307 * @param lastmod The last-modified instant (or <jk>null</jk> to omit). 308 * @param changefreq Change-frequency hint (e.g. {@code "weekly"}, {@code "daily"}; or 309 * <jk>null</jk> to omit). 310 * @param priority Priority value 0.0..1.0 (or <jk>null</jk> to omit). 311 * @return This object. 312 */ 313 public Builder sitemapEntry(String url, ZonedDateTime lastmod, String changefreq, Double priority) { 314 sitemapEntries.add(new SitemapEntry(url, lastmod, changefreq, priority)); 315 return this; 316 } 317 318 /** 319 * Builds a {@link SeoMixin} instance. 320 * 321 * @return A configured instance. 322 */ 323 @Override /* AbstractRestBuilder */ 324 public SeoMixin build() { 325 return new SeoMixin(this); 326 } 327 328 String buildRobots() { 329 if (customRobotsTxt != null) 330 return customRobotsTxt; 331 if (robotsRules.isEmpty()) 332 return DEFAULT_ROBOTS; 333 var sb = new StringBuilder(); 334 for (var r : robotsRules) { 335 sb.append("User-agent: ").append(r.userAgent).append('\n'); 336 for (var p : r.paths) 337 sb.append(r.allow ? "Allow: " : "Disallow: ").append(p).append('\n'); 338 } 339 return sb.toString(); 340 } 341 } 342 343 /** Internal representation of a single user-agent + allow/disallow + paths rule. */ 344 @SuppressWarnings({ 345 "java:S125" // The constructor comment is explanatory prose referencing robotsAllow/robotsDisallow, not commented-out code. 346 }) 347 private static final class RobotsRule { 348 final String userAgent; 349 final boolean allow; 350 final String[] paths; 351 352 RobotsRule(String userAgent, boolean allow, String[] paths) { 353 this.userAgent = userAgent; 354 this.allow = allow; 355 // paths arrives via varargs from robotsAllow/robotsDisallow → never null in practice; 356 // copy to insulate downstream mutation. 357 this.paths = cp(paths); 358 } 359 } 360}