diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts
index f1cf0dee..3ed8a633 100644
--- a/app/robots.txt/route.ts
+++ b/app/robots.txt/route.ts
@@ -1,22 +1,16 @@
-// app/robots.txt/route.ts
+import { getSitemapBaseUrl } from '@/lib/sitemap-utils';
import { isSitemapEnabled } from '@/lib/utils';
export async function GET() {
- const baseUrl = process.env.NEXT_PUBLIC_PLATFORM_URL;
+ const lines = ['User-agent: *', 'Allow: /'];
- const robotsTxt = `User-agent: *
- Allow: /
-
- Sitemap: ${baseUrl}/sitemap/main.xml
- `;
-
- if (!isSitemapEnabled()) {
- return new Response('Sitemaps are not enabled', { status: 404 });
+ if (isSitemapEnabled()) {
+ lines.push('', `Sitemap: ${getSitemapBaseUrl()}/sitemap.xml`);
}
- return new Response(robotsTxt, {
+ return new Response(`${lines.join('\n')}\n`, {
headers: {
- 'Content-Type': 'text/plain',
+ 'Content-Type': 'text/plain; charset=utf-8',
},
});
}
diff --git a/app/sitemap.xml/route.ts b/app/sitemap.xml/route.ts
new file mode 100644
index 00000000..b7ea81ec
--- /dev/null
+++ b/app/sitemap.xml/route.ts
@@ -0,0 +1,41 @@
+import {
+ generateSitemapIndexXml,
+ getChildSitemapUrls,
+} from '@/lib/sitemap-utils';
+import { getSiteMapConfig, isSitemapEnabled } from '@/lib/utils';
+
+export async function GET() {
+ if (!isSitemapEnabled()) {
+ return new Response('Sitemaps are not enabled', { status: 404 });
+ }
+
+ try {
+ const flags = getSiteMapConfig();
+ const childUrls = await getChildSitemapUrls();
+ const sitemapIndex = generateSitemapIndexXml(childUrls);
+
+ return new Response(sitemapIndex, {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/xml; charset=utf-8',
+ 'Cache-Control': `public, max-age=${flags.cacheDuration}`,
+ },
+ });
+ } catch (error) {
+ console.error('Error generating sitemap index:', error);
+
+ return new Response(
+ `
+
+`,
+ {
+ status: 500,
+ headers: {
+ 'Content-Type': 'application/xml; charset=utf-8',
+ },
+ }
+ );
+ }
+}
+
+export const dynamic = 'force-dynamic';
diff --git a/app/sitemap/[entityPage]/route.ts b/app/sitemap/[entityPage]/route.ts
index 54522e49..1193dd8f 100644
--- a/app/sitemap/[entityPage]/route.ts
+++ b/app/sitemap/[entityPage]/route.ts
@@ -56,62 +56,66 @@ function getEntityLoc(
): string {
switch (entity) {
case 'organizations': {
- const orgSlug = item.slug || item.name || item.id;
+ const orgSlug = encodeURIComponent(item.slug || item.name || item.id);
return `${baseUrl}/publishers/organization/${orgSlug}_${item.id}`;
}
case 'users': {
- const userSlug = item.fullName || item.id;
+ const userSlug = encodeURIComponent(item.fullName || item.id);
return `${baseUrl}/publishers/${userSlug}_${item.id}`;
}
case 'collaboratives':
- return `${baseUrl}/collaboratives/${item.slug || item.id}`;
+ return `${baseUrl}/collaboratives/${encodeURIComponent(item.slug || item.id)}`;
case 'datasets':
- return `${baseUrl}/datasets/${item.slug || item.id}`;
+ // App routes use dataset id (not slug).
+ return `${baseUrl}/datasets/${item.id}`;
case 'aimodels':
return `${baseUrl}/aimodels/${item.id}`;
case 'usecases':
- return `${baseUrl}/usecases/${item.slug || item.id}`;
+ // App routes use usecase id (not slug).
+ return `${baseUrl}/usecases/${item.id}`;
case 'sectors':
- return `${baseUrl}/sectors/${item.slug || item.id}`;
+ return `${baseUrl}/sectors/${encodeURIComponent(item.slug || item.id)}`;
default:
- return `${baseUrl}/${path}/${item.slug || item.id}`;
+ return `${baseUrl}/${path}/${encodeURIComponent(item.slug || item.id)}`;
}
}
+function formatLastmod(value: string): string | null {
+ const date = new Date(value);
+ if (Number.isNaN(date.getTime())) return null;
+ return date.toISOString();
+}
+
function generateEntitySitemap(items: EntityItem[], entity: string): string {
const baseUrl = getSitemapBaseUrl();
const config = ENTITY_CONFIG[entity];
if (!config) {
return `
-
- `;
+
+`;
}
const urls = items
?.map((item) => {
const loc = escapeXml(getEntityLoc(baseUrl, entity, item, config.path));
const modifiedAt = item.updated_at || item.updatedAt || item.modified;
- const lastmod = modifiedAt
- ? new Date(modifiedAt).toISOString()
- : new Date().toISOString();
+ const lastmod = modifiedAt ? formatLastmod(modifiedAt) : null;
return `
-
- ${loc}
- ${lastmod}
- weekly
- ${config.priority}
-
- `;
+
+ ${loc}${lastmod ? `\n ${lastmod}` : ''}
+ `;
})
.join('');
- return `\n\n${urls}\n`;
+ return `
+${urls}
+`;
}
export async function GET(
- request: NextRequest,
+ _request: NextRequest,
{ params }: { params: Promise<{ entityPage: string }> }
) {
if (!isSitemapEnabled()) {
@@ -143,7 +147,7 @@ export async function GET(
const flags = getSiteMapConfig();
return new Response(sitemap, {
headers: {
- 'Content-Type': 'application/xml',
+ 'Content-Type': 'application/xml; charset=utf-8',
'Cache-Control': `public, max-age=${flags.childCacheDuration}`,
},
});
@@ -151,13 +155,13 @@ export async function GET(
console.error('Error generating entity sitemap:', error);
const errorSitemap = `
-
- `;
+
+`;
return new Response(errorSitemap, {
status: 500,
headers: {
- 'Content-Type': 'application/xml',
+ 'Content-Type': 'application/xml; charset=utf-8',
},
});
}
diff --git a/app/sitemap/main.xml/route.ts b/app/sitemap/main.xml/route.ts
index 25a116a0..064d92a6 100644
--- a/app/sitemap/main.xml/route.ts
+++ b/app/sitemap/main.xml/route.ts
@@ -1,98 +1,16 @@
-// app/sitemap.xml/route.ts
-import { getAllEntityCounts, getSitemapBaseUrl } from '@/lib/sitemap-utils';
-import { ENTITY_CONFIG, getSiteMapConfig, isSitemapEnabled } from '@/lib/utils';
+import { NextResponse } from 'next/server';
-function generateStaticUrls(baseUrl: string): string {
- const staticPages = [
- { path: '', priority: '1.0', changefreq: 'daily' },
- { path: '/datasets', priority: '0.9', changefreq: 'daily' },
- { path: '/usecases', priority: '0.8', changefreq: 'weekly' },
- { path: '/collaboratives', priority: '0.8', changefreq: 'weekly' },
- { path: '/publishers', priority: '0.7', changefreq: 'weekly' },
- { path: '/sectors', priority: '0.7', changefreq: 'weekly' },
- { path: '/about-us', priority: '0.5', changefreq: 'monthly' },
- ];
-
- return staticPages
- .map(
- (page) => `
-
- ${baseUrl}${page.path}
- ${page.changefreq}
- ${page.priority}
- `
- )
- .join('');
-}
-
-function generateSitemapIndex(
- sitemapUrls: string[],
- staticUrls: string
-): string {
- const sitemapEntries = sitemapUrls
- .map(
- (url) =>
- `
-
- ${url}
- ${new Date().toISOString()}
- `
- )
- .join('');
-
- return `\n
- ${staticUrls}\n${sitemapEntries}\n`;
-}
+import { getSitemapBaseUrl } from '@/lib/sitemap-utils';
+import { isSitemapEnabled } from '@/lib/utils';
+/** Legacy path — redirect to the canonical sitemap index. */
export async function GET() {
if (!isSitemapEnabled()) {
return new Response('Sitemaps are not enabled', { status: 404 });
}
- try {
- const flags = getSiteMapConfig();
- const ITEMS_PER_SITEMAP = flags.itemsPerPage;
- const baseUrl = getSitemapBaseUrl();
-
- const sitemapUrls: string[] = [];
- const entityCounts = await getAllEntityCounts();
-
- Object.keys(ENTITY_CONFIG).forEach((entity) => {
- const count = entityCounts[entity] || 0;
- if (count <= 0) return;
-
- const pages = Math.ceil(count / ITEMS_PER_SITEMAP);
- for (let i = 1; i <= pages; i++) {
- sitemapUrls.push(`${baseUrl}/sitemap/${entity}-${i}.xml`);
- }
- });
-
- const sitemapIndex = generateSitemapIndex(
- sitemapUrls,
- generateStaticUrls(baseUrl)
- );
-
- return new Response(sitemapIndex, {
- status: 200,
- headers: {
- 'Content-Type': 'application/xml',
- 'Cache-Control': `public, max-age=${flags.cacheDuration}`,
- },
- });
- } catch (error) {
- console.error('Error generating sitemap index:', error);
-
- const errorSitemap = `
-
-`;
-
- return new Response(errorSitemap, {
- status: 500,
- headers: {
- 'Content-Type': 'application/xml',
- },
- });
- }
+ const baseUrl = getSitemapBaseUrl();
+ return NextResponse.redirect(`${baseUrl}/sitemap.xml`, 308);
}
export const dynamic = 'force-dynamic';
diff --git a/app/sitemap/static.xml/route.ts b/app/sitemap/static.xml/route.ts
new file mode 100644
index 00000000..aa58665e
--- /dev/null
+++ b/app/sitemap/static.xml/route.ts
@@ -0,0 +1,24 @@
+import {
+ generateStaticSitemapXml,
+ getSitemapBaseUrl,
+} from '@/lib/sitemap-utils';
+import { getSiteMapConfig, isSitemapEnabled } from '@/lib/utils';
+
+export async function GET() {
+ if (!isSitemapEnabled()) {
+ return new Response('Sitemaps are not enabled', { status: 404 });
+ }
+
+ const flags = getSiteMapConfig();
+ const sitemap = generateStaticSitemapXml(getSitemapBaseUrl());
+
+ return new Response(sitemap, {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/xml; charset=utf-8',
+ 'Cache-Control': `public, max-age=${flags.cacheDuration}`,
+ },
+ });
+}
+
+export const dynamic = 'force-dynamic';
diff --git a/lib/sitemap-utils.ts b/lib/sitemap-utils.ts
index 06add03c..cdc8b18a 100644
--- a/lib/sitemap-utils.ts
+++ b/lib/sitemap-utils.ts
@@ -1,11 +1,7 @@
-import { ENTITY_CONFIG, ENTITY_CONFIG_TYPE } from '@/lib/utils';
+import { ENTITY_CONFIG, ENTITY_CONFIG_TYPE, getSiteMapConfig } from '@/lib/utils';
export function getSitemapBaseUrl(): string {
- return (
- process.env.NEXTAUTH_URL ||
- process.env.NEXT_PUBLIC_PLATFORM_URL ||
- ''
- ).replace(/\/$/, '');
+ return (process.env.NEXT_PUBLIC_PLATFORM_URL || '').replace(/\/$/, '');
}
export function escapeXml(value: string): string {
@@ -33,6 +29,7 @@ export async function getGraphqlEntityCount(
query: config.graphqlQuery,
variables: {},
}),
+ cache: 'no-store',
}
);
@@ -84,7 +81,7 @@ export async function getSearchEntityCount(
headers: {
'Content-Type': 'application/json',
},
- next: { revalidate: 3600 },
+ cache: 'no-store',
}
);
@@ -129,3 +126,61 @@ export const getAllEntityCounts = async (): Promise> => {
return counts;
};
+
+export async function getChildSitemapUrls(): Promise {
+ const baseUrl = getSitemapBaseUrl();
+ const itemsPerPage = getSiteMapConfig().itemsPerPage;
+ const urls: string[] = [`${baseUrl}/sitemap/static.xml`];
+
+ const entityCounts = await getAllEntityCounts();
+
+ Object.keys(ENTITY_CONFIG).forEach((entity) => {
+ const count = entityCounts[entity] || 0;
+ if (count <= 0) return;
+
+ const pages = Math.ceil(count / itemsPerPage);
+ for (let i = 1; i <= pages; i++) {
+ urls.push(`${baseUrl}/sitemap/${entity}-${i}.xml`);
+ }
+ });
+
+ return urls;
+}
+
+export function generateSitemapIndexXml(sitemapUrls: string[]): string {
+ const entries = sitemapUrls
+ .map(
+ (url) => `
+
+ ${escapeXml(url)}
+ `
+ )
+ .join('');
+
+ return `
+${entries}
+`;
+}
+
+export const STATIC_SITEMAP_PATHS = [
+ '',
+ '/datasets',
+ '/usecases',
+ '/collaboratives',
+ '/publishers',
+ '/sectors',
+ '/about-us',
+] as const;
+
+export function generateStaticSitemapXml(baseUrl: string): string {
+ const urls = STATIC_SITEMAP_PATHS.map(
+ (path) => `
+
+ ${escapeXml(`${baseUrl}${path}`)}
+ `
+ ).join('');
+
+ return `
+${urls}
+`;
+}
diff --git a/lib/utils.ts b/lib/utils.ts
index 677b2347..ebe9cf52 100644
--- a/lib/utils.ts
+++ b/lib/utils.ts
@@ -179,7 +179,6 @@ export async function getWebsiteTitle(url: string): Promise {
}
// Feature Sitemaps
-// Get configuration from environment
export const getSiteMapConfig = () => ({
itemsPerPage: parseInt(process.env.FEATURE_SITEMAP_ITEMS_PER_PAGE || '1000'),
cacheDuration: parseInt(process.env.FEATURE_SITEMAP_CACHE_DURATION || '3600'),
@@ -202,7 +201,6 @@ export type ENTITY_CONFIG_TYPE = Record<
// Optional filter when a GraphQL union returns mixed types
filterTypename?: 'TypeOrganization' | 'TypeUser';
path: string;
- priority: string;
}
>;
@@ -220,7 +218,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = {
source: 'search',
endpoint: '/search/dataset/',
path: 'datasets',
- priority: '0.8',
},
aimodels: {
source: 'graphql',
@@ -232,7 +229,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = {
}`,
queryResKey: 'aiModels',
path: 'aimodels',
- priority: '0.8',
},
usecases: {
source: 'graphql',
@@ -244,7 +240,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = {
}`,
queryResKey: 'publishedUseCases',
path: 'usecases',
- priority: '0.7',
},
collaboratives: {
source: 'graphql',
@@ -256,7 +251,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = {
}`,
queryResKey: 'publishedCollaboratives',
path: 'collaboratives',
- priority: '0.7',
},
organizations: {
source: 'graphql',
@@ -273,7 +267,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = {
queryResKey: 'getPublishers',
filterTypename: 'TypeOrganization',
path: 'publishers/organization',
- priority: '0.6',
},
users: {
source: 'graphql',
@@ -289,7 +282,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = {
queryResKey: 'getPublishers',
filterTypename: 'TypeUser',
path: 'publishers',
- priority: '0.6',
},
sectors: {
source: 'graphql',
@@ -301,7 +293,6 @@ export const ENTITY_CONFIG: ENTITY_CONFIG_TYPE = {
}`,
queryResKey: 'activeSectors',
path: 'sectors',
- priority: '0.6',
},
};
export const extractPublisherId = (publisherSlug: any) => {