update sitemap

app/robots.txt/route.ts

@@ -1,22 +1,16 @@
-// app/robots.txt/route.ts
+import { getSitemapBaseUrl } from '@/lib/sitemap-utils';
 import { isSitemapEnabled } from '@/lib/utils';
 
 export async function GET() {
-  const baseUrl = process.env.NEXT_PUBLIC_PLATFORM_URL;
+  const lines = ['User-agent: *', 'Allow: /'];
 
-  const robotsTxt = `User-agent: *
-    Allow: /
-
-    Sitemap: ${baseUrl}/sitemap/main.xml
-    `;
-
-  if (!isSitemapEnabled()) {
-    return new Response('Sitemaps are not enabled', { status: 404 });
+  if (isSitemapEnabled()) {
+    lines.push('', `Sitemap: ${getSitemapBaseUrl()}/sitemap.xml`);
   }
 
-  return new Response(robotsTxt, {
+  return new Response(`${lines.join('\n')}\n`, {
     headers: {
-      'Content-Type': 'text/plain',
+      'Content-Type': 'text/plain; charset=utf-8',
     },
   });
 }

app/sitemap.xml/route.ts

@@ -0,0 +1,41 @@
+import {
+  generateSitemapIndexXml,
+  getChildSitemapUrls,
+} from '@/lib/sitemap-utils';
+import { getSiteMapConfig, isSitemapEnabled } from '@/lib/utils';
+
+export async function GET() {
+  if (!isSitemapEnabled()) {
+    return new Response('Sitemaps are not enabled', { status: 404 });
+  }
+
+  try {
+    const flags = getSiteMapConfig();
+    const childUrls = await getChildSitemapUrls();
+    const sitemapIndex = generateSitemapIndexXml(childUrls);
+
+    return new Response(sitemapIndex, {
+      status: 200,
+      headers: {
+        'Content-Type': 'application/xml; charset=utf-8',
+        'Cache-Control': `public, max-age=${flags.cacheDuration}`,
+      },
+    });
+  } catch (error) {
+    console.error('Error generating sitemap index:', error);
+
+    return new Response(
+      `<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+</sitemapindex>`,
+      {
+        status: 500,
+        headers: {
+          'Content-Type': 'application/xml; charset=utf-8',
+        },
+      }
+    );
+  }
+}
+
+export const dynamic = 'force-dynamic';

app/sitemap/[entityPage]/route.ts

@@ -56,62 +56,66 @@ function getEntityLoc(
 ): string {
   switch (entity) {
     case 'organizations': {
-      const orgSlug = item.slug || item.name || item.id;
+      const orgSlug = encodeURIComponent(item.slug || item.name || item.id);
       return `${baseUrl}/publishers/organization/${orgSlug}_${item.id}`;
     }
     case 'users': {
-      const userSlug = item.fullName || item.id;
+      const userSlug = encodeURIComponent(item.fullName || item.id);
       return `${baseUrl}/publishers/${userSlug}_${item.id}`;
     }
     case 'collaboratives':
-      return `${baseUrl}/collaboratives/${item.slug || item.id}`;
+      return `${baseUrl}/collaboratives/${encodeURIComponent(item.slug || item.id)}`;
     case 'datasets':
-      return `${baseUrl}/datasets/${item.slug || item.id}`;
+      // App routes use dataset id (not slug).
+      return `${baseUrl}/datasets/${item.id}`;
     case 'aimodels':
       return `${baseUrl}/aimodels/${item.id}`;
     case 'usecases':
-      return `${baseUrl}/usecases/${item.slug || item.id}`;
+      // App routes use usecase id (not slug).
+      return `${baseUrl}/usecases/${item.id}`;
     case 'sectors':
-      return `${baseUrl}/sectors/${item.slug || item.id}`;
+      return `${baseUrl}/sectors/${encodeURIComponent(item.slug || item.id)}`;
     default:
-      return `${baseUrl}/${path}/${item.slug || item.id}`;
+      return `${baseUrl}/${path}/${encodeURIComponent(item.slug || item.id)}`;
   }
 }
 
+function formatLastmod(value: string): string | null {
+  const date = new Date(value);
+  if (Number.isNaN(date.getTime())) return null;
+  return date.toISOString();
+}
+
 function generateEntitySitemap(items: EntityItem[], entity: string): string {
   const baseUrl = getSitemapBaseUrl();
   const config = ENTITY_CONFIG[entity];
 
   if (!config) {
     return `<?xml version="1.0" encoding="UTF-8"?>
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-    </urlset>`;
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+</urlset>`;
   }
 
   const urls = items
     ?.map((item) => {
       const loc = escapeXml(getEntityLoc(baseUrl, entity, item, config.path));
       const modifiedAt = item.updated_at || item.updatedAt || item.modified;
-      const lastmod = modifiedAt
-        ? new Date(modifiedAt).toISOString()
-        : new Date().toISOString();
+      const lastmod = modifiedAt ? formatLastmod(modifiedAt) : null;
 
       return `
-        <url>
-          <loc>${loc}</loc>
-          <lastmod>${lastmod}</lastmod>
-          <changefreq>weekly</changefreq>
-          <priority>${config.priority}</priority>
-        </url>
-      `;
+  <url>
+    <loc>${loc}</loc>${lastmod ? `\n    <lastmod>${lastmod}</lastmod>` : ''}
+  </url>`;
     })
     .join('');
 
-  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>`;
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">${urls}
+</urlset>`;
 }
 
 export async function GET(
-  request: NextRequest,
+  _request: NextRequest,
   { params }: { params: Promise<{ entityPage: string }> }
 ) {
   if (!isSitemapEnabled()) {
@@ -143,21 +147,21 @@ export async function GET(
     const flags = getSiteMapConfig();
     return new Response(sitemap, {
       headers: {
-        'Content-Type': 'application/xml',
+        'Content-Type': 'application/xml; charset=utf-8',
         'Cache-Control': `public, max-age=${flags.childCacheDuration}`,
       },
     });
   } catch (error) {
     console.error('Error generating entity sitemap:', error);
 
     const errorSitemap = `<?xml version="1.0" encoding="UTF-8"?>
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-    </urlset>`;
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+</urlset>`;
 
     return new Response(errorSitemap, {
       status: 500,
       headers: {
-        'Content-Type': 'application/xml',
+        'Content-Type': 'application/xml; charset=utf-8',
       },
     });
   }

app/sitemap/main.xml/route.ts

@@ -1,98 +1,16 @@
-// app/sitemap.xml/route.ts
-import { getAllEntityCounts, getSitemapBaseUrl } from '@/lib/sitemap-utils';
-import { ENTITY_CONFIG, getSiteMapConfig, isSitemapEnabled } from '@/lib/utils';
+import { NextResponse } from 'next/server';
 
-function generateStaticUrls(baseUrl: string): string {
-  const staticPages = [
-    { path: '', priority: '1.0', changefreq: 'daily' },
-    { path: '/datasets', priority: '0.9', changefreq: 'daily' },
-    { path: '/usecases', priority: '0.8', changefreq: 'weekly' },
-    { path: '/collaboratives', priority: '0.8', changefreq: 'weekly' },
-    { path: '/publishers', priority: '0.7', changefreq: 'weekly' },
-    { path: '/sectors', priority: '0.7', changefreq: 'weekly' },
-    { path: '/about-us', priority: '0.5', changefreq: 'monthly' },
-  ];
-
-  return staticPages
-    .map(
-      (page) => `
-    <url>
-      <loc>${baseUrl}${page.path}</loc>
-      <changefreq>${page.changefreq}</changefreq>
-      <priority>${page.priority}</priority>
-    </url>`
-    )
-    .join('');
-}
-
-function generateSitemapIndex(
-  sitemapUrls: string[],
-  staticUrls: string
-): string {
-  const sitemapEntries = sitemapUrls
-    .map(
-      (url) =>
-        `
-    <url>
-      <loc>${url}</loc>
-      <lastmod>${new Date().toISOString()}</lastmod>
-    </url>`
-    )
-    .join('');
-
-  return `<?xml version="1.0" encoding="UTF-8"?>\n
-  <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">${staticUrls}\n${sitemapEntries}\n</urlset>`;
-}
+import { getSitemapBaseUrl } from '@/lib/sitemap-utils';
+import { isSitemapEnabled } from '@/lib/utils';
 
+/** Legacy path — redirect to the canonical sitemap index. */
 export async function GET() {
   if (!isSitemapEnabled()) {
     return new Response('Sitemaps are not enabled', { status: 404 });
   }
 
-  try {
-    const flags = getSiteMapConfig();
-    const ITEMS_PER_SITEMAP = flags.itemsPerPage;
-    const baseUrl = getSitemapBaseUrl();
-
-    const sitemapUrls: string[] = [];
-    const entityCounts = await getAllEntityCounts();
-
-    Object.keys(ENTITY_CONFIG).forEach((entity) => {
-      const count = entityCounts[entity] || 0;
-      if (count <= 0) return;
-
-      const pages = Math.ceil(count / ITEMS_PER_SITEMAP);
-      for (let i = 1; i <= pages; i++) {
-        sitemapUrls.push(`${baseUrl}/sitemap/${entity}-${i}.xml`);
-      }
-    });
-
-    const sitemapIndex = generateSitemapIndex(
-      sitemapUrls,
-      generateStaticUrls(baseUrl)
-    );
-
-    return new Response(sitemapIndex, {
-      status: 200,
-      headers: {
-        'Content-Type': 'application/xml',
-        'Cache-Control': `public, max-age=${flags.cacheDuration}`,
-      },
-    });
-  } catch (error) {
-    console.error('Error generating sitemap index:', error);
-
-    const errorSitemap = `<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-</urlset>`;
-
-    return new Response(errorSitemap, {
-      status: 500,
-      headers: {
-        'Content-Type': 'application/xml',
-      },
-    });
-  }
+  const baseUrl = getSitemapBaseUrl();
+  return NextResponse.redirect(`${baseUrl}/sitemap.xml`, 308);
 }
 
 export const dynamic = 'force-dynamic';

app/sitemap/static.xml/route.ts

@@ -0,0 +1,24 @@
+import {
+  generateStaticSitemapXml,
+  getSitemapBaseUrl,
+} from '@/lib/sitemap-utils';
+import { getSiteMapConfig, isSitemapEnabled } from '@/lib/utils';
+
+export async function GET() {
+  if (!isSitemapEnabled()) {
+    return new Response('Sitemaps are not enabled', { status: 404 });
+  }
+
+  const flags = getSiteMapConfig();
+  const sitemap = generateStaticSitemapXml(getSitemapBaseUrl());
+
+  return new Response(sitemap, {
+    status: 200,
+    headers: {
+      'Content-Type': 'application/xml; charset=utf-8',
+      'Cache-Control': `public, max-age=${flags.cacheDuration}`,
+    },
+  });
+}
+
+export const dynamic = 'force-dynamic';