diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts index f1cf0dee..3ed8a633 100644 --- a/app/robots.txt/route.ts +++ b/app/robots.txt/route.ts @@ -1,22 +1,16 @@ -// app/robots.txt/route.ts +import { getSitemapBaseUrl } from '@/lib/sitemap-utils'; import { isSitemapEnabled } from '@/lib/utils'; export async function GET() { - const baseUrl = process.env.NEXT_PUBLIC_PLATFORM_URL; + const lines = ['User-agent: *', 'Allow: /']; - const robotsTxt = `User-agent: * - Allow: / - - Sitemap: ${baseUrl}/sitemap/main.xml - `; - - if (!isSitemapEnabled()) { - return new Response('Sitemaps are not enabled', { status: 404 }); + if (isSitemapEnabled()) { + lines.push('', `Sitemap: ${getSitemapBaseUrl()}/sitemap.xml`); } - return new Response(robotsTxt, { + return new Response(`${lines.join('\n')}\n`, { headers: { - 'Content-Type': 'text/plain', + 'Content-Type': 'text/plain; charset=utf-8', }, }); } diff --git a/app/sitemap.xml/route.ts b/app/sitemap.xml/route.ts new file mode 100644 index 00000000..b7ea81ec --- /dev/null +++ b/app/sitemap.xml/route.ts @@ -0,0 +1,41 @@ +import { + generateSitemapIndexXml, + getChildSitemapUrls, +} from '@/lib/sitemap-utils'; +import { getSiteMapConfig, isSitemapEnabled } from '@/lib/utils'; + +export async function GET() { + if (!isSitemapEnabled()) { + return new Response('Sitemaps are not enabled', { status: 404 }); + } + + try { + const flags = getSiteMapConfig(); + const childUrls = await getChildSitemapUrls(); + const sitemapIndex = generateSitemapIndexXml(childUrls); + + return new Response(sitemapIndex, { + status: 200, + headers: { + 'Content-Type': 'application/xml; charset=utf-8', + 'Cache-Control': `public, max-age=${flags.cacheDuration}`, + }, + }); + } catch (error) { + console.error('Error generating sitemap index:', error); + + return new Response( + ` + +`, + { + status: 500, + headers: { + 'Content-Type': 'application/xml; charset=utf-8', + }, + } + ); + } +} + +export const dynamic = 'force-dynamic'; diff --git a/app/sitemap/[entityPage]/route.ts b/app/sitemap/[entityPage]/route.ts index 54522e49..1193dd8f 100644 --- a/app/sitemap/[entityPage]/route.ts +++ b/app/sitemap/[entityPage]/route.ts @@ -56,62 +56,66 @@ function getEntityLoc( ): string { switch (entity) { case 'organizations': { - const orgSlug = item.slug || item.name || item.id; + const orgSlug = encodeURIComponent(item.slug || item.name || item.id); return `${baseUrl}/publishers/organization/${orgSlug}_${item.id}`; } case 'users': { - const userSlug = item.fullName || item.id; + const userSlug = encodeURIComponent(item.fullName || item.id); return `${baseUrl}/publishers/${userSlug}_${item.id}`; } case 'collaboratives': - return `${baseUrl}/collaboratives/${item.slug || item.id}`; + return `${baseUrl}/collaboratives/${encodeURIComponent(item.slug || item.id)}`; case 'datasets': - return `${baseUrl}/datasets/${item.slug || item.id}`; + // App routes use dataset id (not slug). + return `${baseUrl}/datasets/${item.id}`; case 'aimodels': return `${baseUrl}/aimodels/${item.id}`; case 'usecases': - return `${baseUrl}/usecases/${item.slug || item.id}`; + // App routes use usecase id (not slug). + return `${baseUrl}/usecases/${item.id}`; case 'sectors': - return `${baseUrl}/sectors/${item.slug || item.id}`; + return `${baseUrl}/sectors/${encodeURIComponent(item.slug || item.id)}`; default: - return `${baseUrl}/${path}/${item.slug || item.id}`; + return `${baseUrl}/${path}/${encodeURIComponent(item.slug || item.id)}`; } } +function formatLastmod(value: string): string | null { + const date = new Date(value); + if (Number.isNaN(date.getTime())) return null; + return date.toISOString(); +} + function generateEntitySitemap(items: EntityItem[], entity: string): string { const baseUrl = getSitemapBaseUrl(); const config = ENTITY_CONFIG[entity]; if (!config) { return ` - - `; + +`; } const urls = items ?.map((item) => { const loc = escapeXml(getEntityLoc(baseUrl, entity, item, config.path)); const modifiedAt = item.updated_at || item.updatedAt || item.modified; - const lastmod = modifiedAt - ? new Date(modifiedAt).toISOString() - : new Date().toISOString(); + const lastmod = modifiedAt ? formatLastmod(modifiedAt) : null; return ` - - ${loc} - ${lastmod} - weekly - ${config.priority} - - `; + + ${loc}${lastmod ? `\n ${lastmod}` : ''} + `; }) .join(''); - return `\n\n${urls}\n`; + return ` +${urls} +`; } export async function GET( - request: NextRequest, + _request: NextRequest, { params }: { params: Promise<{ entityPage: string }> } ) { if (!isSitemapEnabled()) { @@ -143,7 +147,7 @@ export async function GET( const flags = getSiteMapConfig(); return new Response(sitemap, { headers: { - 'Content-Type': 'application/xml', + 'Content-Type': 'application/xml; charset=utf-8', 'Cache-Control': `public, max-age=${flags.childCacheDuration}`, }, }); @@ -151,13 +155,13 @@ export async function GET( console.error('Error generating entity sitemap:', error); const errorSitemap = ` - - `; + +`; return new Response(errorSitemap, { status: 500, headers: { - 'Content-Type': 'application/xml', + 'Content-Type': 'application/xml; charset=utf-8', }, }); } diff --git a/app/sitemap/main.xml/route.ts b/app/sitemap/main.xml/route.ts index 25a116a0..064d92a6 100644 --- a/app/sitemap/main.xml/route.ts +++ b/app/sitemap/main.xml/route.ts @@ -1,98 +1,16 @@ -// app/sitemap.xml/route.ts -import { getAllEntityCounts, getSitemapBaseUrl } from '@/lib/sitemap-utils'; -import { ENTITY_CONFIG, getSiteMapConfig, isSitemapEnabled } from '@/lib/utils'; +import { NextResponse } from 'next/server'; -function generateStaticUrls(baseUrl: string): string { - const staticPages = [ - { path: '', priority: '1.0', changefreq: 'daily' }, - { path: '/datasets', priority: '0.9', changefreq: 'daily' }, - { path: '/usecases', priority: '0.8', changefreq: 'weekly' }, - { path: '/collaboratives', priority: '0.8', changefreq: 'weekly' }, - { path: '/publishers', priority: '0.7', changefreq: 'weekly' }, - { path: '/sectors', priority: '0.7', changefreq: 'weekly' }, - { path: '/about-us', priority: '0.5', changefreq: 'monthly' }, - ]; - - return staticPages - .map( - (page) => ` - - ${baseUrl}${page.path} - ${page.changefreq} - ${page.priority} - ` - ) - .join(''); -} - -function generateSitemapIndex( - sitemapUrls: string[], - staticUrls: string -): string { - const sitemapEntries = sitemapUrls - .map( - (url) => - ` - - ${url} - ${new Date().toISOString()} - ` - ) - .join(''); - - return `\n - ${staticUrls}\n${sitemapEntries}\n`; -} +import { getSitemapBaseUrl } from '@/lib/sitemap-utils'; +import { isSitemapEnabled } from '@/lib/utils'; +/** Legacy path — redirect to the canonical sitemap index. */ export async function GET() { if (!isSitemapEnabled()) { return new Response('Sitemaps are not enabled', { status: 404 }); } - try { - const flags = getSiteMapConfig(); - const ITEMS_PER_SITEMAP = flags.itemsPerPage; - const baseUrl = getSitemapBaseUrl(); - - const sitemapUrls: string[] = []; - const entityCounts = await getAllEntityCounts(); - - Object.keys(ENTITY_CONFIG).forEach((entity) => { - const count = entityCounts[entity] || 0; - if (count <= 0) return; - - const pages = Math.ceil(count / ITEMS_PER_SITEMAP); - for (let i = 1; i <= pages; i++) { - sitemapUrls.push(`${baseUrl}/sitemap/${entity}-${i}.xml`); - } - }); - - const sitemapIndex = generateSitemapIndex( - sitemapUrls, - generateStaticUrls(baseUrl) - ); - - return new Response(sitemapIndex, { - status: 200, - headers: { - 'Content-Type': 'application/xml', - 'Cache-Control': `public, max-age=${flags.cacheDuration}`, - }, - }); - } catch (error) { - console.error('Error generating sitemap index:', error); - - const errorSitemap = ` - -`; - - return new Response(errorSitemap, { - status: 500, - headers: { - 'Content-Type': 'application/xml', - }, - }); - } + const baseUrl = getSitemapBaseUrl(); + return NextResponse.redirect(`${baseUrl}/sitemap.xml`, 308); } export const dynamic = 'force-dynamic'; diff --git a/app/sitemap/static.xml/route.ts b/app/sitemap/static.xml/route.ts new file mode 100644 index 00000000..aa58665e --- /dev/null +++ b/app/sitemap/static.xml/route.ts @@ -0,0 +1,24 @@ +import { + generateStaticSitemapXml, + getSitemapBaseUrl, +} from '@/lib/sitemap-utils'; +import { getSiteMapConfig, isSitemapEnabled } from '@/lib/utils'; + +export async function GET() { + if (!isSitemapEnabled()) { + return new Response('Sitemaps are not enabled', { status: 404 }); + } + + const flags = getSiteMapConfig(); + const sitemap = generateStaticSitemapXml(getSitemapBaseUrl()); + + return new Response(sitemap, { + status: 200, + headers: { + 'Content-Type': 'application/xml; charset=utf-8', + 'Cache-Control': `public, max-age=${flags.cacheDuration}`, + }, + }); +} + +export const dynamic = 'force-dynamic'; diff --git a/lib/sitemap-utils.ts b/lib/sitemap-utils.ts index 06add03c..cdc8b18a 100644 --- a/lib/sitemap-utils.ts +++ b/lib/sitemap-utils.ts @@ -1,11 +1,7 @@ -import { ENTITY_CONFIG, ENTITY_CONFIG_TYPE } from '@/lib/utils'; +import { ENTITY_CONFIG, ENTITY_CONFIG_TYPE, getSiteMapConfig } from '@/lib/utils'; export function getSitemapBaseUrl(): string { - return ( - process.env.NEXTAUTH_URL || - process.env.NEXT_PUBLIC_PLATFORM_URL || - '' - ).replace(/\/$/, ''); + return (process.env.NEXT_PUBLIC_PLATFORM_URL || '').replace(/\/$/, ''); } export function escapeXml(value: string): string { @@ -33,6 +29,7 @@ export async function getGraphqlEntityCount( query: config.graphqlQuery, variables: {}, }), + cache: 'no-store', } ); @@ -84,7 +81,7 @@ export async function getSearchEntityCount( headers: { 'Content-Type': 'application/json', }, - next: { revalidate: 3600 }, + cache: 'no-store', } ); @@ -129,3 +126,61 @@ export const getAllEntityCounts = async (): Promise> => { return counts; }; + +export async function getChildSitemapUrls(): Promise { + const baseUrl = getSitemapBaseUrl(); + const itemsPerPage = getSiteMapConfig().itemsPerPage; + const urls: string[] = [`${baseUrl}/sitemap/static.xml`]; + + const entityCounts = await getAllEntityCounts(); + + Object.keys(ENTITY_CONFIG).forEach((entity) => { + const count = entityCounts[entity] || 0; + if (count <= 0) return; + + const pages = Math.ceil(count / itemsPerPage); + for (let i = 1; i <= pages; i++) { + urls.push(`${baseUrl}/sitemap/${entity}-${i}.xml`); + } + }); + + return urls; +} + +export function generateSitemapIndexXml(sitemapUrls: string[]): string { + const entries = sitemapUrls + .map( + (url) => ` + + ${escapeXml(url)} + ` + ) + .join(''); + + return ` +${entries} +`; +} + +export const STATIC_SITEMAP_PATHS = [ + '', + '/datasets', + '/usecases', + '/collaboratives', + '/publishers', + '/sectors', + '/about-us', +] as const; + +export function generateStaticSitemapXml(baseUrl: string): string { + const urls = STATIC_SITEMAP_PATHS.map( + (path) => ` + + ${escapeXml(`${baseUrl}${path}`)} + ` + ).join(''); + + return ` +${urls} +`; +} diff --git a/lib/utils.ts b/lib/utils.ts index 677b2347..ebe9cf52 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -179,7 +179,6 @@ export async function getWebsiteTitle(url: string): Promise { } // Feature Sitemaps -// Get configuration from environment export const getSiteMapConfig = () => ({ itemsPerPage: parseInt(process.env.FEATURE_SITEMAP_ITEMS_PER_PAGE || '1000'), cacheDuration: parseInt(process.env.FEATURE_SITEMAP_CACHE_DURATION || '3600'), @@ -202,7 +201,6 @@ export type ENTITY_CONFIG_TYPE = Record< // Optional filter when a GraphQL union returns mixed types filterTypename?: 'TypeOrganization' | 'TypeUser'; path: string; - priority: string; } >; @@ -220,7 +218,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = { source: 'search', endpoint: '/search/dataset/', path: 'datasets', - priority: '0.8', }, aimodels: { source: 'graphql', @@ -232,7 +229,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = { }`, queryResKey: 'aiModels', path: 'aimodels', - priority: '0.8', }, usecases: { source: 'graphql', @@ -244,7 +240,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = { }`, queryResKey: 'publishedUseCases', path: 'usecases', - priority: '0.7', }, collaboratives: { source: 'graphql', @@ -256,7 +251,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = { }`, queryResKey: 'publishedCollaboratives', path: 'collaboratives', - priority: '0.7', }, organizations: { source: 'graphql', @@ -273,7 +267,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = { queryResKey: 'getPublishers', filterTypename: 'TypeOrganization', path: 'publishers/organization', - priority: '0.6', }, users: { source: 'graphql', @@ -289,7 +282,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = { queryResKey: 'getPublishers', filterTypename: 'TypeUser', path: 'publishers', - priority: '0.6', }, sectors: { source: 'graphql', @@ -301,7 +293,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = { }`, queryResKey: 'activeSectors', path: 'sectors', - priority: '0.6', }, }; export const extractPublisherId = (publisherSlug: any) => {