Port upstream search, thumbnail, geometry, and cache improvements

.env.example

@@ -17,6 +17,8 @@ DB_POOL_MAX=10
 # Redis / caching
 ENDPOINT_CACHE=true
 CACHE_DEBUG_HEADERS=false
+# Opt in only after all cache readers support the OGM response codec.
+CACHE_REDIS_COMPRESSION_ENABLED=false
 REDIS_HOST=localhost
 REDIS_PORT=6380
 REDIS_HOST_PORT=6380

backend/app/api/v1/endpoint_modules/map.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import Optional
+from typing import Literal, Optional
 
 from fastapi import APIRouter, HTTPException, Query, Request
 from fastapi.responses import JSONResponse
@@ -25,6 +25,13 @@
 async def map_h3(
     request: Request,
     q: Optional[str] = Query(None, description="Search query"),
+    include_filter_operator: Literal["and", "or"] = Query(
+        "or",
+        description=(
+            "How repeated values within one include filter are combined; "
+            "use 'and' for drill-down faceting"
+        ),
+    ),
     adv_q: Optional[str] = Query(
         None,
         description=(
@@ -67,6 +74,7 @@ async def map_h3(
             q=q,
             fq=fq or None,
             include_filters=include_filters or None,
+            include_filter_operator=include_filter_operator,
             exclude_filters=exclude_filters or None,
             adv_q=parsed_adv_q,
             bbox=bbox,

backend/app/api/v1/endpoint_modules/search.py

@@ -3,7 +3,7 @@
 import logging
 import os
 import time
-from typing import Annotated, Optional
+from typing import Annotated, Literal, Optional
 
 from fastapi import APIRouter, Body, HTTPException, Query, Request
 from fastapi.responses import JSONResponse
@@ -21,10 +21,12 @@
     sanitize_for_json,
 )
 from app.elasticsearch.search import (
+    DEFAULT_INCLUDE_FILTER_OPERATOR,
     generate_facet_apply_template,
     get_facet_aggregation_config,
     get_facet_values,
     get_search_criteria,
+    normalize_include_filter_operator,
     process_facet_response,
 )
 from app.services.allmaps_service import fetch_allmaps_attributes_map
@@ -196,6 +198,7 @@ def _build_semantic_search_cache_key(
     fields,
     facets,
     include_filters,
+    include_filter_operator=DEFAULT_INCLUDE_FILTER_OPERATOR,
     exclude_filters,
     fq,
     adv_q,
@@ -213,6 +216,7 @@ def _build_semantic_search_cache_key(
         fields=fields or "",
         facets=facets or "",
         include_filters=_canonical_filter_value(include_filters or {}),
+        include_filter_operator=include_filter_operator,
         exclude_filters=_canonical_filter_value(exclude_filters or {}),
         fq=_canonical_filter_value(fq or {}),
         adv_q=adv_q or [],
@@ -408,6 +412,9 @@ async def _handle_search(request: Request, params: dict) -> JSONResponse:
     callback = params.get("callback")
     request_query_params = params.get("request_query_params")
     include_filters = params.get("include_filters")
+    include_filter_operator = normalize_include_filter_operator(
+        params.get("include_filter_operator")
+    )
     exclude_filters = params.get("exclude_filters")
     fq = params.get("fq")
     adv_q = params.get("adv_q")
@@ -430,6 +437,7 @@ async def _handle_search(request: Request, params: dict) -> JSONResponse:
         fields=fields,
         facets=facets,
         include_filters=include_filters,
+        include_filter_operator=include_filter_operator,
         exclude_filters=exclude_filters,
         fq=fq,
         adv_q=adv_q,
@@ -474,6 +482,7 @@ async def _handle_search(request: Request, params: dict) -> JSONResponse:
         callback=callback,
         facets=facets,
         include_filters=include_filters,
+        include_filter_operator=include_filter_operator,
         exclude_filters=exclude_filters,
         fq_direct=fq,
         adv_q=adv_q,
@@ -769,6 +778,10 @@ async def search(
             "Each clause: {'op': 'AND|OR|NOT', 'f': 'dct_title_s', 'q': 'Iowa'}"
         ),
     ),
+    include_filter_operator: Literal["and", "or"] = Query(
+        DEFAULT_INCLUDE_FILTER_OPERATOR,
+        description="Combine repeated filter values with 'and' for drill-down; defaults to 'or'.",
+    ),
 ):
     """Search resources."""
 
@@ -841,6 +854,7 @@ async def search(
                 "adv_q": parsed_adv_q,
                 "fq": filter_query,
                 "include_filters": include_filters,
+                "include_filter_operator": include_filter_operator,
                 "exclude_filters": exclude_filters,
             },
         )
@@ -887,6 +901,7 @@ async def search_post(
     Supported keys:
       - q, page, per_page, sort, search_field, fields, facets, meta
       - include_filters, exclude_filters, fq (object of field->values)
+      - include_filter_operator ("and" for drill-down, "or" by default)
       - adv_q (array of query clauses with op, f, q)
     """
 
@@ -903,6 +918,17 @@ async def search_post(
     adv_q = payload.get("adv_q")
 
     include_filters = payload.get("include_filters")
+    include_filter_operator = str(
+        payload.get("include_filter_operator", DEFAULT_INCLUDE_FILTER_OPERATOR)
+    ).lower()
+    if include_filter_operator not in {"and", "or"}:
+        return api_error_response(
+            status_code=400,
+            code="invalid_include_filter_operator",
+            title="Bad request",
+            detail="include_filter_operator must be 'and' or 'or'",
+            request_id=get_request_id(request),
+        )
     exclude_filters = payload.get("exclude_filters")
     fq = payload.get("fq")
 
@@ -921,6 +947,7 @@ async def search_post(
                 "meta": meta,
                 "callback": callback,
                 "include_filters": include_filters,
+                "include_filter_operator": include_filter_operator,
                 "exclude_filters": exclude_filters,
                 "fq": fq,
                 "adv_q": adv_q,
@@ -957,6 +984,10 @@ async def get_facet(
             "Each clause: {'op': 'AND|OR|NOT', 'f': 'dct_title_s', 'q': 'Iowa'}"
         ),
     ),
+    include_filter_operator: Literal["and", "or"] = Query(
+        DEFAULT_INCLUDE_FILTER_OPERATOR,
+        description="Combine repeated filter values with 'and' for drill-down; defaults to 'or'.",
+    ),
 ):
     """Get paginated, sortable facet values for a specific facet field within a search resultset.
 
@@ -1029,6 +1060,7 @@ async def get_facet(
             query=q,
             fq=filter_query,
             include_filters=include_filters,
+            include_filter_operator=include_filter_operator,
             exclude_filters=exclude_filters,
             adv_q=parsed_adv_q,
             q_facet=q_facet,
@@ -1059,6 +1091,7 @@ async def get_facet(
             {
                 "q": q,
                 "include_filters": include_filters,
+                "include_filter_operator": include_filter_operator,
                 "exclude_filters": exclude_filters,
                 "fq": filter_query,
                 "adv_q": parsed_adv_q,

backend/app/api/v1/strong_params.py

@@ -15,6 +15,7 @@
     "fields",  # Field filtering for response attributes
     "facets",  # Facet filtering for response aggregations
     "meta",  # Include per-resource meta
+    "include_filter_operator",  # Repeated exact filter values: and/or
     "callback",  # JSONP callback
     # Explicit facet filter parameters (fq[<field>][]) expected by tests
     "fq[dct_resourceClass_sm][]",
@@ -50,6 +51,7 @@
     "sort",  # Sort option (count_desc, count_asc, alpha_asc, alpha_desc)
     "q_facet",  # Search query to filter facet values
     "adv_q",  # Advanced multi-field search queries
+    "include_filter_operator",  # Repeated exact filter values: and/or
     "callback",  # JSONP callback
     # Explicit facet filter parameters (fq[<field>][]) expected by tests
     "fq[dct_resourceClass_sm][]",

backend/app/elasticsearch/index.py

@@ -17,6 +17,7 @@
 from shapely.geometry import mapping as shapely_mapping
 
 from app.services.language_service import ensure_b1g_language
+from app.services.temporal_normalization import normalize_or_derive_index_year
 from db.database import database
 from db.models import ogm_resource_state, resources
 
@@ -357,6 +358,10 @@ async def process_resource(resource_dict):
             processed_dict[key] = value
 
     ensure_b1g_language(processed_dict)
+    if "gbl_indexYear_im" in processed_dict or "gbl_dateRange_drsim" in processed_dict:
+        processed_dict["gbl_indexYear_im"] = normalize_or_derive_index_year(
+            processed_dict.get("gbl_indexYear_im"), processed_dict.get("gbl_dateRange_drsim")
+        )
 
     explicit_ogm_repo_values = _coerce_ogm_repo_values(processed_dict.get("ogm_repo"))
     if explicit_ogm_repo_values: