Easiest way to add robots.txt and sitemap.xml in Next Js to improve SEO

We will start with the simpler one, robots.txt

Is really simple, the file is written in the body. The URL is whatever you want, you can get that from an ENV var as well.

//src/app/robots.txt/route.js
export async function GET() {
const URL = "https://windmotion.io";
const body = `User-agent: *
Disallow:

Sitemap: ${URL}/sitemap.xml
`;
  return new Response(body, {
    status: 200,
    headers: {
      "Cache-control": "public, s-maxage=86400, stale-while-revalidate",
      "content-type": "text/plain",
    },
  });
}

Create SiteMap with a blog

Create a way to add articles hrefs, probably using mdx.

It's as simple as having articles have an href, or an id. Then this will generate the sitemap dynamically.

//src/app/sitemap.xml/route.js
import { loadArticles } from '@/lib/mdx'

function generateSiteMap(articles) {
  const URL = "https://windmotion.io";

  return `<?xml version="1.0" encoding="UTF-8"?>
   <urlset xmlns="https://www.sitemaps.org/schemas/sitemap/0.9">
     <!-- Add the static URLs manually -->
     <url>
       <loc>${URL}</loc>
     </url>
     <url>
       <loc>${URL}/about</loc>
     </url>
      <url>
       <loc>${URL}/blog</loc>
     </url>
     <url>
       <loc>${URL}/work</loc>
     </url>
     <url>
       <loc>${URL}/contact</loc>
     </url>
     ${articles
       .map(({ href }) => {
         return `
           <url>
               <loc>${`${URL}/blog/${href}`}</loc>
           </url>
         `;
       })
       .join("")}
   </urlset>
 `;
}

export async function GET() {
  const articles = await loadArticles()
  const body = generateSiteMap(articles);
 
  return new Response(body, {
    status: 200,
    headers: {
      "Cache-control": "public, s-maxage=86400, stale-while-revalidate",
      "content-type": "application/xml",
    },
  });
}

Bye!