diff --git a/.env.example b/.env.example index 9a44a71..92bac14 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,8 @@ DB_POOL_MAX=10 # Redis / caching ENDPOINT_CACHE=true CACHE_DEBUG_HEADERS=false +# Opt in only after all cache readers support the OGM response codec. +CACHE_REDIS_COMPRESSION_ENABLED=false REDIS_HOST=localhost REDIS_PORT=6380 REDIS_HOST_PORT=6380 diff --git a/backend/app/api/v1/endpoint_modules/map.py b/backend/app/api/v1/endpoint_modules/map.py index e1eadda..932276e 100644 --- a/backend/app/api/v1/endpoint_modules/map.py +++ b/backend/app/api/v1/endpoint_modules/map.py @@ -1,6 +1,6 @@ import json import logging -from typing import Optional +from typing import Literal, Optional from fastapi import APIRouter, HTTPException, Query, Request from fastapi.responses import JSONResponse @@ -25,6 +25,13 @@ async def map_h3( request: Request, q: Optional[str] = Query(None, description="Search query"), + include_filter_operator: Literal["and", "or"] = Query( + "or", + description=( + "How repeated values within one include filter are combined; " + "use 'and' for drill-down faceting" + ), + ), adv_q: Optional[str] = Query( None, description=( @@ -67,6 +74,7 @@ async def map_h3( q=q, fq=fq or None, include_filters=include_filters or None, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters or None, adv_q=parsed_adv_q, bbox=bbox, diff --git a/backend/app/api/v1/endpoint_modules/search.py b/backend/app/api/v1/endpoint_modules/search.py index 5ca57c1..02dec43 100644 --- a/backend/app/api/v1/endpoint_modules/search.py +++ b/backend/app/api/v1/endpoint_modules/search.py @@ -3,7 +3,7 @@ import logging import os import time -from typing import Annotated, Optional +from typing import Annotated, Literal, Optional from fastapi import APIRouter, Body, HTTPException, Query, Request from fastapi.responses import JSONResponse @@ -21,10 +21,12 @@ sanitize_for_json, ) from app.elasticsearch.search import ( + DEFAULT_INCLUDE_FILTER_OPERATOR, generate_facet_apply_template, get_facet_aggregation_config, get_facet_values, get_search_criteria, + normalize_include_filter_operator, process_facet_response, ) from app.services.allmaps_service import fetch_allmaps_attributes_map @@ -196,6 +198,7 @@ def _build_semantic_search_cache_key( fields, facets, include_filters, + include_filter_operator=DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters, fq, adv_q, @@ -213,6 +216,7 @@ def _build_semantic_search_cache_key( fields=fields or "", facets=facets or "", include_filters=_canonical_filter_value(include_filters or {}), + include_filter_operator=include_filter_operator, exclude_filters=_canonical_filter_value(exclude_filters or {}), fq=_canonical_filter_value(fq or {}), adv_q=adv_q or [], @@ -408,6 +412,9 @@ async def _handle_search(request: Request, params: dict) -> JSONResponse: callback = params.get("callback") request_query_params = params.get("request_query_params") include_filters = params.get("include_filters") + include_filter_operator = normalize_include_filter_operator( + params.get("include_filter_operator") + ) exclude_filters = params.get("exclude_filters") fq = params.get("fq") adv_q = params.get("adv_q") @@ -430,6 +437,7 @@ async def _handle_search(request: Request, params: dict) -> JSONResponse: fields=fields, facets=facets, include_filters=include_filters, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters, fq=fq, adv_q=adv_q, @@ -474,6 +482,7 @@ async def _handle_search(request: Request, params: dict) -> JSONResponse: callback=callback, facets=facets, include_filters=include_filters, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters, fq_direct=fq, adv_q=adv_q, @@ -769,6 +778,10 @@ async def search( "Each clause: {'op': 'AND|OR|NOT', 'f': 'dct_title_s', 'q': 'Iowa'}" ), ), + include_filter_operator: Literal["and", "or"] = Query( + DEFAULT_INCLUDE_FILTER_OPERATOR, + description="Combine repeated filter values with 'and' for drill-down; defaults to 'or'.", + ), ): """Search resources.""" @@ -841,6 +854,7 @@ async def search( "adv_q": parsed_adv_q, "fq": filter_query, "include_filters": include_filters, + "include_filter_operator": include_filter_operator, "exclude_filters": exclude_filters, }, ) @@ -887,6 +901,7 @@ async def search_post( Supported keys: - q, page, per_page, sort, search_field, fields, facets, meta - include_filters, exclude_filters, fq (object of field->values) + - include_filter_operator ("and" for drill-down, "or" by default) - adv_q (array of query clauses with op, f, q) """ @@ -903,6 +918,17 @@ async def search_post( adv_q = payload.get("adv_q") include_filters = payload.get("include_filters") + include_filter_operator = str( + payload.get("include_filter_operator", DEFAULT_INCLUDE_FILTER_OPERATOR) + ).lower() + if include_filter_operator not in {"and", "or"}: + return api_error_response( + status_code=400, + code="invalid_include_filter_operator", + title="Bad request", + detail="include_filter_operator must be 'and' or 'or'", + request_id=get_request_id(request), + ) exclude_filters = payload.get("exclude_filters") fq = payload.get("fq") @@ -921,6 +947,7 @@ async def search_post( "meta": meta, "callback": callback, "include_filters": include_filters, + "include_filter_operator": include_filter_operator, "exclude_filters": exclude_filters, "fq": fq, "adv_q": adv_q, @@ -957,6 +984,10 @@ async def get_facet( "Each clause: {'op': 'AND|OR|NOT', 'f': 'dct_title_s', 'q': 'Iowa'}" ), ), + include_filter_operator: Literal["and", "or"] = Query( + DEFAULT_INCLUDE_FILTER_OPERATOR, + description="Combine repeated filter values with 'and' for drill-down; defaults to 'or'.", + ), ): """Get paginated, sortable facet values for a specific facet field within a search resultset. @@ -1029,6 +1060,7 @@ async def get_facet( query=q, fq=filter_query, include_filters=include_filters, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters, adv_q=parsed_adv_q, q_facet=q_facet, @@ -1059,6 +1091,7 @@ async def get_facet( { "q": q, "include_filters": include_filters, + "include_filter_operator": include_filter_operator, "exclude_filters": exclude_filters, "fq": filter_query, "adv_q": parsed_adv_q, diff --git a/backend/app/api/v1/strong_params.py b/backend/app/api/v1/strong_params.py index f6b4245..d1c6988 100644 --- a/backend/app/api/v1/strong_params.py +++ b/backend/app/api/v1/strong_params.py @@ -15,6 +15,7 @@ "fields", # Field filtering for response attributes "facets", # Facet filtering for response aggregations "meta", # Include per-resource meta + "include_filter_operator", # Repeated exact filter values: and/or "callback", # JSONP callback # Explicit facet filter parameters (fq[][]) expected by tests "fq[dct_resourceClass_sm][]", @@ -50,6 +51,7 @@ "sort", # Sort option (count_desc, count_asc, alpha_asc, alpha_desc) "q_facet", # Search query to filter facet values "adv_q", # Advanced multi-field search queries + "include_filter_operator", # Repeated exact filter values: and/or "callback", # JSONP callback # Explicit facet filter parameters (fq[][]) expected by tests "fq[dct_resourceClass_sm][]", diff --git a/backend/app/elasticsearch/index.py b/backend/app/elasticsearch/index.py index 8562539..7385a30 100644 --- a/backend/app/elasticsearch/index.py +++ b/backend/app/elasticsearch/index.py @@ -17,6 +17,7 @@ from shapely.geometry import mapping as shapely_mapping from app.services.language_service import ensure_b1g_language +from app.services.temporal_normalization import normalize_or_derive_index_year from db.database import database from db.models import ogm_resource_state, resources @@ -357,6 +358,10 @@ async def process_resource(resource_dict): processed_dict[key] = value ensure_b1g_language(processed_dict) + if "gbl_indexYear_im" in processed_dict or "gbl_dateRange_drsim" in processed_dict: + processed_dict["gbl_indexYear_im"] = normalize_or_derive_index_year( + processed_dict.get("gbl_indexYear_im"), processed_dict.get("gbl_dateRange_drsim") + ) explicit_ogm_repo_values = _coerce_ogm_repo_values(processed_dict.get("ogm_repo")) if explicit_ogm_repo_values: diff --git a/backend/app/elasticsearch/search.py b/backend/app/elasticsearch/search.py index f5a74c6..e07e2fd 100644 --- a/backend/app/elasticsearch/search.py +++ b/backend/app/elasticsearch/search.py @@ -5,7 +5,7 @@ import re import time from dataclasses import dataclass -from typing import Optional +from typing import Literal, Optional from urllib.parse import urlencode from dotenv import load_dotenv @@ -50,6 +50,13 @@ SEARCH_TIMING_LOG_THRESHOLD_MS = float(os.getenv("SEARCH_TIMING_LOG_THRESHOLD_MS", "750")) SEARCH_FACET_CACHE_NAMESPACE = "search.facets" FACET_VALUES_CACHE_NAMESPACE = "search.facet_values" +IncludeFilterOperator = Literal["and", "or"] +DEFAULT_INCLUDE_FILTER_OPERATOR: IncludeFilterOperator = "or" + + +def normalize_include_filter_operator(value: str | None) -> IncludeFilterOperator: + """Preserve historical OR semantics unless a client explicitly requests AND.""" + return "and" if str(value or "").lower() == "and" else "or" def _escape_query_string_brackets(query_text: str) -> str: @@ -130,6 +137,27 @@ def _resolve_filter_field(field: str) -> str: return field +def _build_exact_filter_clauses( + field: str, + values, + include_filter_operator: IncludeFilterOperator = DEFAULT_INCLUDE_FILTER_OPERATOR, +) -> list[dict]: + """Build exact-match clauses for one filter field. + + OR preserves the API's historical ``terms`` behavior. AND emits one + ``term`` clause per selected value; sibling clauses in ``bool.filter`` are + conjunctive, which produces drill-down faceting for multi-valued fields. + """ + resolved_field = _resolve_filter_field(field) + if not isinstance(values, list): + return [{"term": {resolved_field: values}}] + if not values: + return [] + if include_filter_operator == "and": + return [{"term": {resolved_field: value}} for value in values] + return [{"terms": {resolved_field: values}}] + + def get_facet_aggregation_config(facet_name: str) -> dict: """Get Elasticsearch aggregation configuration for a given facet name. @@ -313,6 +341,7 @@ def _build_search_facet_cache_key( search_fields: str | None, fq: dict | None, include_filters: dict | None, + include_filter_operator: IncludeFilterOperator = DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters: dict | None, adv_q: Optional[list], selected_aggs: tuple[str, ...], @@ -324,6 +353,7 @@ def _build_search_facet_cache_key( search_fields=_normalize_search_fields(search_fields), fq=fq or {}, include_filters=include_filters or {}, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters or {}, adv_q=adv_q or [], aggs=selected_aggs, @@ -337,6 +367,7 @@ def _build_facet_values_cache_key( query: str | None, fq: dict | None, include_filters: dict | None, + include_filter_operator: IncludeFilterOperator = DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters: dict | None, adv_q: Optional[list], q_facet: str | None, @@ -349,6 +380,7 @@ def _build_facet_values_cache_key( query=query or "", fq=fq or {}, include_filters=include_filters or {}, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters or {}, adv_q=adv_q or [], q_facet=q_facet or "", @@ -1112,6 +1144,7 @@ class SearchParams: sort: list | None = None search_fields: str | None = None include_filters: dict | None = None + include_filter_operator: IncludeFilterOperator = DEFAULT_INCLUDE_FILTER_OPERATOR exclude_filters: dict | None = None facets: str | None = None adv_q: list | None = None @@ -1129,6 +1162,7 @@ def from_inputs( sort: list | None, search_fields: str | None, include_filters: dict | None, + include_filter_operator: str | None = DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters: dict | None, facets: str | None, adv_q: list | None, @@ -1143,6 +1177,7 @@ def from_inputs( sort=sort, search_fields=search_fields, include_filters=include_filters, + include_filter_operator=normalize_include_filter_operator(include_filter_operator), exclude_filters=exclude_filters, facets=facets, adv_q=adv_q, @@ -1196,6 +1231,7 @@ async def prepare(self, params: SearchParams, search_criteria: dict) -> SearchFa search_fields=params.search_fields, fq=params.fq, include_filters=params.include_filters, + include_filter_operator=params.include_filter_operator, exclude_filters=params.exclude_filters, adv_q=params.adv_q, selected_aggs=selected_agg_names, @@ -1333,10 +1369,13 @@ def _build_filters(self) -> SearchFilterPlan: f"Processing filter - Field: {field}, " f"Resolved: {resolved_field}, Values: {values}" ) - if isinstance(values, list): - filter_clauses.append({"terms": {resolved_field: values}}) - else: - filter_clauses.append({"term": {resolved_field: values}}) + filter_clauses.extend( + _build_exact_filter_clauses( + field, + values, + self.params.include_filter_operator, + ) + ) if self.params.include_filters: for field, values in self.params.include_filters.items(): @@ -1356,7 +1395,13 @@ def _build_filters(self) -> SearchFilterPlan: if values and str(values[0]).lower() == "true": filter_clauses.append({"term": {resolved_field: True}}) elif isinstance(values, list): - filter_clauses.append({"terms": {resolved_field: values}}) + filter_clauses.extend( + _build_exact_filter_clauses( + field, + values, + self.params.include_filter_operator, + ) + ) else: filter_clauses.append({"term": {resolved_field: values}}) @@ -1825,6 +1870,7 @@ async def build(self, execution: SearchExecutionResult) -> dict: self.search_criteria, overlap_context=execution.overlap_context, include_filters=self.params.include_filters, + include_filter_operator=self.params.include_filter_operator, exclude_filters=self.params.exclude_filters, adv_q=self.params.adv_q, hydrate_hits=self.params.hydrate_hits, @@ -1859,6 +1905,7 @@ async def search_resources( sort: list = None, search_fields: str | None = None, include_filters: dict | None = None, + include_filter_operator: str | None = DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters: dict | None = None, facets: Optional[str] = None, adv_q: Optional[list] = None, @@ -1874,6 +1921,7 @@ async def search_resources( sort=sort, search_fields=search_fields, include_filters=include_filters, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters, facets=facets, adv_q=adv_q, @@ -1996,6 +2044,7 @@ async def process_search_response( search_criteria, overlap_context: dict | None = None, include_filters: dict | None = None, + include_filter_operator: IncludeFilterOperator = DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters: dict | None = None, adv_q: Optional[list] = None, hydrate_hits: bool = True, @@ -2175,6 +2224,7 @@ def _compute_spatial_metrics(hit_dict: dict, ctx: dict) -> dict[str, float] | No { "q": search_criteria.get("query"), "include_filters": include_filters, + "include_filter_operator": include_filter_operator, "exclude_filters": exclude_filters, "fq": search_criteria.get("filters"), "adv_q": adv_q, @@ -2232,6 +2282,7 @@ async def map_h3_aggregation( q: Optional[str] = None, fq: Optional[dict] = None, include_filters: Optional[dict] = None, + include_filter_operator: str | None = DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters: Optional[dict] = None, adv_q: Optional[list] = None, bbox: Optional[str] = None, @@ -2245,16 +2296,15 @@ async def map_h3_aggregation( index_name = os.getenv("ELASTICSEARCH_INDEX", "opengeometadata_api") if resolution < 2 or resolution > 8: resolution = 5 + normalized_filter_operator = normalize_include_filter_operator(include_filter_operator) filter_clauses = [] must_not_clauses = [] if fq: for field, values in fq.items(): - resolved = _resolve_filter_field(field) - if isinstance(values, list): - filter_clauses.append({"terms": {resolved: values}}) - else: - filter_clauses.append({"term": {resolved: values}}) + filter_clauses.extend( + _build_exact_filter_clauses(field, values, normalized_filter_operator) + ) if include_filters: # Apply location (bbox) filter so hex counts match the search results @@ -2281,7 +2331,9 @@ async def map_h3_aggregation( if yr["range"]["gbl_indexYear_im"]: filter_clauses.append(yr) elif isinstance(values, list): - filter_clauses.append({"terms": {resolved: values}}) + filter_clauses.extend( + _build_exact_filter_clauses(field, values, normalized_filter_operator) + ) else: filter_clauses.append({"term": {resolved: values}}) @@ -2497,7 +2549,12 @@ def generate_facet_apply_template(facet_id: str, search_context: dict) -> str: fq = (search_context or {}).get("fq") or {} adv_q = (search_context or {}).get("adv_q") - query_params: dict[str, list[str] | str] = {"q": q} + query_params: dict[str, list[str] | str] = { + "q": q, + "include_filter_operator": normalize_include_filter_operator( + (search_context or {}).get("include_filter_operator") + ), + } # Preserve advanced query clauses if present (as a compact JSON string) if adv_q: @@ -2679,6 +2736,7 @@ async def get_facet_values( query: str = None, fq: dict = None, include_filters: dict | None = None, + include_filter_operator: str | None = DEFAULT_INCLUDE_FILTER_OPERATOR, exclude_filters: dict | None = None, adv_q: Optional[list] = None, q_facet: Optional[str] = None, @@ -2712,16 +2770,15 @@ async def get_facet_values( agg_field = facet_config["field"] # Build the same filter query structure as search_resources + normalized_filter_operator = normalize_include_filter_operator(include_filter_operator) filter_clauses = [] must_not_clauses = [] if fq: for field, values in fq.items(): - resolved_field = _resolve_filter_field(field) - if isinstance(values, list): - filter_clauses.append({"terms": {resolved_field: values}}) - else: - filter_clauses.append({"term": {resolved_field: values}}) + filter_clauses.extend( + _build_exact_filter_clauses(field, values, normalized_filter_operator) + ) if include_filters: for field, values in include_filters.items(): @@ -2751,9 +2808,9 @@ async def get_facet_values( if values and str(values[0]).lower() == "true": filter_clauses.append({"term": {resolved_field: True}}) elif isinstance(values, list): - # Use terms to match if ANY of the specified values are present - # This matches the behavior of legacy fq filters (OR logic) - filter_clauses.append({"terms": {resolved_field: values}}) + filter_clauses.extend( + _build_exact_filter_clauses(field, values, normalized_filter_operator) + ) else: filter_clauses.append({"term": {resolved_field: values}}) @@ -2845,6 +2902,7 @@ async def get_facet_values( query=query, fq=fq, include_filters=include_filters, + include_filter_operator=normalized_filter_operator, exclude_filters=exclude_filters, adv_q=adv_q, q_facet=q_facet, diff --git a/backend/app/services/cache_service.py b/backend/app/services/cache_service.py index 289a7e6..ef44772 100644 --- a/backend/app/services/cache_service.py +++ b/backend/app/services/cache_service.py @@ -24,6 +24,7 @@ get_durable_api_response, store_durable_api_response, ) +from app.services.response_cache_codec import decode_response_record, encode_response_record # Load environment variables from .env file load_dotenv() @@ -40,6 +41,9 @@ ENDPOINT_CACHE = os.getenv("ENDPOINT_CACHE", "false").lower() == "true" CACHE_DEBUG_HEADERS = os.getenv("CACHE_DEBUG_HEADERS", "false").lower() == "true" CACHE_LOG_EVENTS = os.getenv("CACHE_LOG_EVENTS", "false").lower() == "true" +CACHE_REDIS_COMPRESSION_ENABLED = ( + os.getenv("CACHE_REDIS_COMPRESSION_ENABLED", "false").lower() == "true" +) # Default cache expiration (12 hours) DEFAULT_CACHE_TTL = int(os.getenv("CACHE_TTL", 43200)) @@ -475,7 +479,7 @@ async def get_record(self, key: str) -> Optional[dict[str, Any]]: try: raw = await _redis_call(self._redis_client.get(key)) if raw: - record = json.loads(raw) + record = decode_response_record(raw) if isinstance(record, dict) and record.get("schema") == RECORD_SCHEMA_VERSION: return record except Exception as e: @@ -519,7 +523,7 @@ async def set_record( redis_ok = False try: if self._redis_client: - raw = json.dumps(record, separators=(",", ":"), sort_keys=True).encode("utf-8") + raw = encode_response_record(record, compress=CACHE_REDIS_COMPRESSION_ENABLED) redis_ok = bool(await _redis_call(self._redis_client.set(key, raw, ex=ttl_seconds))) except Exception as e: logger.error(f"Error setting record cache: {str(e)}") diff --git a/backend/app/services/image_service.py b/backend/app/services/image_service.py index f6dd319..2ac40f5 100644 --- a/backend/app/services/image_service.py +++ b/backend/app/services/image_service.py @@ -216,21 +216,24 @@ def _extract_thumbnail_from_manifest_json( self.logger.debug(f"Found manifest-level thumbnail: {candidate}") return self._standardize_iiif_url(candidate) - # Sequences - Prefer direct resource @id, then service @id + # IIIF v2: the resource ID may be an HTML catalog page (e.g. OSU). + # Prefer the declared image service to construct a bounded rendition. if manifest_json.get("sequences"): self.logger.debug("Image: sequences") canvas = manifest_json.get("sequences", [{}])[0].get("canvases", [{}])[0] image = canvas.get("images", [{}])[0].get("resource", {}) - - # Prefer direct image ID when present + service = image.get("service") + if isinstance(service, list): + service = service[0] if service else None + if isinstance(service, dict): + service_id = service.get("@id") or service.get("id") + else: + service_id = service if isinstance(service, str) else None + if service_id: + return self._standardize_iiif_url(service_id, image_service=True) if image.get("@id"): return self._standardize_iiif_url(image["@id"]) - # Fallback to image service @id to construct consistent size - service_id = image.get("service", {}).get("@id") - if service_id: - return self._standardize_iiif_url(service_id) - # Items - IIIF v3 style elif manifest_json.get("items"): # Check for thumbnail in first canvas (items[0].thumbnail) @@ -272,7 +275,7 @@ def _extract_thumbnail_from_manifest_json( if body_service_id: self.logger.debug(f"Found body service ID: {body_service_id}") - return self._standardize_iiif_url(body_service_id) + return self._standardize_iiif_url(body_service_id, image_service=True) # Next try body.id (prefer direct ID unmodified) if body.get("id"): @@ -402,12 +405,16 @@ def get_iiif_image_thumbnail(self, info_url: str) -> Optional[str]: return None return self._extract_thumbnail_from_iiif_info_json(info_json, info_url) - def _standardize_iiif_url(self, url: str) -> str: + def _standardize_iiif_url(self, url: str, *, image_service: bool = False) -> str: """ Standardize IIIF image URLs to ensure consistent size. Converts various IIIF image URLs to a standard bounded-box rendition. """ try: + # A declared service may have no /iiif/ path (for example Loris). + if image_service: + return f"{url.rstrip('/').removesuffix('/info.json')}{IIIF_THUMBNAIL_PATH}" + # Skip if not a likely IIIF URL if not any(x in url.lower() for x in ["/iiif/", "/image/", "info.json"]): return url @@ -878,24 +885,14 @@ def _get_thumbnail_source_url( if not iiif_url: continue - # Transform ContentDM IIIF URLs + # Normalize legacy CONTENTdm paths without changing the provider. if url_hostname_matches(iiif_url, "contentdm.oclc.org"): - # Handle both /digital/iiif/ and /iiif/ patterns - # Pattern 1: /digital/iiif/collection/id - match = re.search(r"/digital/iiif/([^/]+)/(\d+)", iiif_url) - if match: - collection, item_id = match.groups() - return self._standardize_iiif_url( - f"https://cdm16022.contentdm.oclc.org/iiif/2/{collection}:{item_id}" - ) - - # Pattern 2: /iiif/collection:id/manifest.json or /iiif/collection:id/ - match = re.search(r"/iiif/([^/]+)/", iiif_url) - if match: - collection_item = match.group(1) - return self._standardize_iiif_url( - f"https://cdm16022.contentdm.oclc.org/iiif/2/{collection_item}" - ) + iiif_url = re.sub( + r"/digital/iiif/([^/]+)/(\d+)(?=/|$)", + r"/iiif/2/\1:\2", + iiif_url, + count=1, + ) # Preserve Image API info documents for the worker. Level 0 services # must be read before choosing one of their advertised static sizes. @@ -920,28 +917,8 @@ def _get_thumbnail_source_url( break if manifest_url: - # Special case: ContentDM manifest URLs can be directly converted to image URLs - # without fetching the manifest, since we know the pattern - if ( - url_hostname_matches(manifest_url, "contentdm.oclc.org") - and "/iiif/" in manifest_url - ): - # Extract collection:item from ContentDM manifest URL - # Pattern: https://cdm16022.contentdm.oclc.org/iiif/p16022coll55:1755/manifest.json - match = re.search(r"/iiif/([^/]+)/", manifest_url) - if match: - collection_item = match.group(1) - # Convert to direct IIIF image URL - image_url = self._standardize_iiif_url( - f"https://cdm16022.contentdm.oclc.org/iiif/2/{collection_item}" - ) - self.logger.info( - f"✅ Directly converted ContentDM manifest to image URL: {image_url}" - ) - return image_url - - # For other manifests, return the source without side effects. The - # thumbnail endpoint owns queueing so each request creates at most one job. + # Compound object IDs need not identify an image. Return the manifest + # without queueing; the thumbnail endpoint owns the worker job. return manifest_url # Use curated b1g_image_ss only after exhausting IIIF-based options. diff --git a/backend/app/services/ogm_harvest/importer.py b/backend/app/services/ogm_harvest/importer.py index e966638..38cf51d 100644 --- a/backend/app/services/ogm_harvest/importer.py +++ b/backend/app/services/ogm_harvest/importer.py @@ -20,6 +20,7 @@ sync_relationships_for_batch, sync_relationships_for_resource_ids, ) +from app.services.temporal_normalization import normalize_or_derive_index_year from db.database import database from db.models import resources @@ -187,15 +188,14 @@ def _normalize_record(self, record: Dict[str, Any], repo_name: str) -> Dict[str, if name in record: out[name] = record.get(name) - # Normalize gbl_indexYear_im to list[int] (db column is ARRAY(Integer)) - if "gbl_indexYear_im" in out: - v = out.get("gbl_indexYear_im") - if isinstance(v, list): - out["gbl_indexYear_im"] = [int(x) for x in v if str(x).isdigit()] or None - elif isinstance(v, (int, str)) and str(v).isdigit(): - out["gbl_indexYear_im"] = [int(v)] - else: - out["gbl_indexYear_im"] = None + # The legacy application derived the index year from the first date range. + # Preserve an explicitly supplied value, but restore that behavior when it + # is missing (the bridge and some OGM feeds only provide the date range). + if "gbl_indexYear_im" in out or "gbl_dateRange_drsim" in out: + out["gbl_indexYear_im"] = normalize_or_derive_index_year( + out.get("gbl_indexYear_im"), + out.get("gbl_dateRange_drsim"), + ) # Array-ish fields: ensure list for Postgres array columns array_fields = { diff --git a/backend/app/services/response_cache_codec.py b/backend/app/services/response_cache_codec.py new file mode 100644 index 0000000..1636f62 --- /dev/null +++ b/backend/app/services/response_cache_codec.py @@ -0,0 +1,30 @@ +"""Versioned Redis encoding; durable records and HTTP bodies remain unchanged.""" + +import json +import zlib +from typing import Any + +COMPRESSION_PREFIX = b"OGM-RC\x01" +MIN_COMPRESSION_BYTES = 4096 +MAX_DECOMPRESSED_BYTES = 16 * 1024 * 1024 + + +def encode_response_record(record: dict[str, Any], *, compress: bool = False) -> bytes: + raw = json.dumps(record, separators=(",", ":"), sort_keys=True).encode("utf-8") + if compress and MIN_COMPRESSION_BYTES <= len(raw) <= MAX_DECOMPRESSED_BYTES: + encoded = COMPRESSION_PREFIX + zlib.compress(raw, level=1) + if len(encoded) <= len(raw) * 0.9: + return encoded + return raw + + +def decode_response_record(raw: bytes | str) -> Any: + if isinstance(raw, bytes) and raw.startswith(COMPRESSION_PREFIX): + decoder = zlib.decompressobj() + decoded = decoder.decompress(raw[len(COMPRESSION_PREFIX) :], MAX_DECOMPRESSED_BYTES + 1) + if len(decoded) > MAX_DECOMPRESSED_BYTES: + raise ValueError("Compressed response record exceeds size limit") + if not decoder.eof or decoder.unused_data or decoder.unconsumed_tail: + raise ValueError("Invalid compressed response record") + raw = decoded + return json.loads(raw) diff --git a/backend/app/services/search_service.py b/backend/app/services/search_service.py index 843244b..6ca4bc3 100644 --- a/backend/app/services/search_service.py +++ b/backend/app/services/search_service.py @@ -113,6 +113,7 @@ async def search( callback: Optional[str] = None, facets: Optional[str] = None, include_filters: Optional[Dict] = None, + include_filter_operator: str = "or", exclude_filters: Optional[Dict] = None, fq_direct: Optional[Dict] = None, adv_q: Optional[list] = None, @@ -176,6 +177,7 @@ async def search( sort=sort_mapping, search_fields=search_fields, include_filters=include_filters, + include_filter_operator=include_filter_operator, exclude_filters=exclude_filters, facets=facets, adv_q=adv_q, @@ -483,22 +485,10 @@ def extract_new_style_filters(self, params: Optional[str]) -> tuple[Dict, Dict]: "extract_new_style_filters: Parsing params: %s...", params[:200] if params else "None", ) - # parse_qs expects a URL-decoded query string - # If params is URL-encoded (contains %5B for [), decode it first - from urllib.parse import unquote - - if params and "%5B" in params: - # URL-encoded brackets detected, decode first - decoded_params = unquote(params) - logger.debug( - "extract_new_style_filters: Decoded params sample: %s", - decoded_params[:200], - ) - raw_params = parse_qs(decoded_params) - elif isinstance(params, str): - raw_params = parse_qs(params) - else: - raw_params = parse_qs(str(params)) + # parse_qs decodes both parameter names and values. Decoding the full + # query string first would turn an encoded value such as ``%26`` into + # a structural ``&`` separator before parsing it. + raw_params = parse_qs(str(params)) geo_keys: list[str] = [] if logger.isEnabledFor(logging.DEBUG): geo_keys = [k for k in raw_params.keys() if "geo" in k.lower()] @@ -650,7 +640,10 @@ def extract_new_style_filters(self, params: Optional[str]) -> tuple[Dict, Dict]: # Handle year_range filters year_range_filters = {} for key, values in raw_params.items(): - if key.startswith("include_filters[year_range][") and key.endswith("]"): + if key in { + "include_filters[year_range][start]", + "include_filters[year_range][end]", + }: sub_key = key[len("include_filters[year_range][") : -1] # start or end year_range_filters[sub_key] = values[0] if values else None @@ -663,6 +656,7 @@ def extract_new_style_filters(self, params: Optional[str]) -> tuple[Dict, Dict]: key.startswith("include_filters[") and key.endswith("][]") and not key.startswith("include_filters[geo][") + and not key.startswith("include_filters[year_range][") ): field = key[len("include_filters[") : -len("][]")] include_filters[field] = values diff --git a/backend/app/services/static_map_service.py b/backend/app/services/static_map_service.py index 684de87..043ff5e 100644 --- a/backend/app/services/static_map_service.py +++ b/backend/app/services/static_map_service.py @@ -51,6 +51,27 @@ ) +class _WebMercatorLine(staticmaps.Line): + """Render source segments directly instead of replacing them with geodesics. + + py-staticmaps projects tiles and coordinates with Web Mercator, but its Line + implementation first interpolates segments of at least one degree longitude + along WGS84 geodesics. Leaflet projects the supplied GeoJSON vertices and + connects them directly, so the interpolation makes the same geometry appear + bowed or loop around the antimeridian in static maps. + """ + + def interpolate(self) -> List[Any]: + return self._latlngs + + +class _WebMercatorArea(staticmaps.Area): + """Area counterpart to _WebMercatorLine with direct projected segments.""" + + def interpolate(self) -> List[Any]: + return self._latlngs + + class StaticMapService: """Service for generating static maps from bounding boxes.""" @@ -151,11 +172,16 @@ def _is_valid_bbox(self, xmin: float, ymin: float, xmax: float, ymax: float) -> if not (-90 <= ymin <= 90) or not (-90 <= ymax <= 90): return False - # Check for valid min/max relationships - if xmin >= xmax or ymin >= ymax: + # Check for valid min/max relationships. A zero-area extent in both + # dimensions is a legitimate point geometry. + if xmin > xmax or ymin > ymax: return False - # Check for zero-area bounding boxes + if xmin == xmax and ymin == ymax: + return True + + # Very thin one-dimensional or near-zero rectangles are not useful + # fallback map extents. if (xmax - xmin) < 0.001 or (ymax - ymin) < 0.001: return False @@ -299,8 +325,8 @@ def _extract_bbox_from_geojson( _STROKE_GLOW_WIDTH = 5 _LINE_WIDTH = 3 _TRANSPARENT_COLOR = staticmaps.Color(0, 0, 0, 0) - _MAP_VARIANT = "static_map_v7" - _BASEMAP_VARIANT = "static_basemap_v5" + _MAP_VARIANT = "static_map_v9" + _BASEMAP_VARIANT = "static_basemap_v7" _ASSET_KEY_PREFIX = "static_map_asset" _ALIAS_KEY_PREFIX = "static_map_alias" _HASH_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) @@ -363,21 +389,13 @@ def _global_map_context(self) -> staticmaps.Context: def _bbox_points(self, bbox_coords: Tuple[float, float, float, float]) -> list: """Convert bbox coords into a closed polygon usable by py-staticmaps.""" xmin, ymin, xmax, ymax = bbox_coords - bbox_width_degrees = xmax - xmin - num_segments = max(50, int(bbox_width_degrees * 2)) - points = [] - points.append(staticmaps.create_latlng(ymin, xmin)) - points.append(staticmaps.create_latlng(ymax, xmin)) - for i in range(1, num_segments): - lon = xmin + (xmax - xmin) * (i / num_segments) - points.append(staticmaps.create_latlng(ymax, lon)) - points.append(staticmaps.create_latlng(ymax, xmax)) - points.append(staticmaps.create_latlng(ymin, xmax)) - for i in range(num_segments - 1, 0, -1): - lon = xmin + (xmax - xmin) * (i / num_segments) - points.append(staticmaps.create_latlng(ymin, lon)) - points.append(points[0]) - return points + return [ + staticmaps.create_latlng(ymin, xmin), + staticmaps.create_latlng(ymax, xmin), + staticmaps.create_latlng(ymax, xmax), + staticmaps.create_latlng(ymin, xmax), + staticmaps.create_latlng(ymin, xmin), + ] def _bbox_area( self, @@ -388,7 +406,7 @@ def _bbox_area( width: int, ) -> Any: """Create a py-staticmaps polygon for the bbox.""" - return staticmaps.Area( + return _WebMercatorArea( self._bbox_points(bbox_coords), fill_color=fill_color, color=color, @@ -976,7 +994,20 @@ def coord_to_latlngs(coord_list: list) -> list: return [staticmaps.create_latlng(float(lat), float(lon)) for lon, lat in coord_list] try: - if geom_type == "Polygon": + if geom_type == "Point": + if len(coordinates) < 2: + return None + lon, lat = float(coordinates[0]), float(coordinates[1]) + marker_size = 10 if include_glow else 0 + objects.append( + staticmaps.Marker( + staticmaps.create_latlng(lat, lon), + color=stroke_color, + size=marker_size, + ) + ) + + elif geom_type == "Polygon": # coordinates: [ exterior_ring, hole1, ... ]; ring is [ [lon,lat], ... ] for ring in coordinates: if len(ring) < 3: @@ -987,14 +1018,14 @@ def coord_to_latlngs(coord_list: list) -> list: points.append(points[0]) # Glow layer first (when requested), then main area if include_glow: - glow_area = staticmaps.Area( + glow_area = _WebMercatorArea( points, fill_color=self._TRANSPARENT_COLOR, color=self._STROKE_GLOW_COLOR, width=self._STROKE_GLOW_WIDTH, ) objects.append(glow_area) - area = staticmaps.Area( + area = _WebMercatorArea( points, fill_color=fill_color, color=stroke_color, @@ -1012,14 +1043,14 @@ def coord_to_latlngs(coord_list: list) -> list: if len(points) > 1: points.append(points[0]) if include_glow: - glow_area = staticmaps.Area( + glow_area = _WebMercatorArea( points, fill_color=self._TRANSPARENT_COLOR, color=self._STROKE_GLOW_COLOR, width=self._STROKE_GLOW_WIDTH, ) objects.append(glow_area) - area = staticmaps.Area( + area = _WebMercatorArea( points, fill_color=fill_color, color=stroke_color, @@ -1030,22 +1061,15 @@ def coord_to_latlngs(coord_list: list) -> list: # has no dashed stroke, so we draw it solid) bbox = self._extract_bbox_from_geojson(geojson) if bbox: - xmin, ymin, xmax, ymax = bbox - extent_points = [ - staticmaps.create_latlng(ymin, xmin), - staticmaps.create_latlng(ymax, xmin), - staticmaps.create_latlng(ymax, xmax), - staticmaps.create_latlng(ymin, xmax), - staticmaps.create_latlng(ymin, xmin), - ] + extent_points = self._bbox_points(bbox) if include_glow: - glow_line = staticmaps.Line( + glow_line = _WebMercatorLine( extent_points, color=self._STROKE_GLOW_COLOR, width=self._STROKE_GLOW_WIDTH, ) objects.append(glow_line) - extent_line = staticmaps.Line( + extent_line = _WebMercatorLine( extent_points, color=stroke_color, width=width, @@ -1057,13 +1081,13 @@ def coord_to_latlngs(coord_list: list) -> list: return None points = coord_to_latlngs(coordinates) if include_glow: - glow_line = staticmaps.Line( + glow_line = _WebMercatorLine( points, color=self._STROKE_GLOW_COLOR, width=self._STROKE_GLOW_WIDTH, ) objects.append(glow_line) - line = staticmaps.Line( + line = _WebMercatorLine( points, color=stroke_color, width=width, @@ -1076,13 +1100,13 @@ def coord_to_latlngs(coord_list: list) -> list: continue points = coord_to_latlngs(line_coords) if include_glow: - glow_line = staticmaps.Line( + glow_line = _WebMercatorLine( points, color=self._STROKE_GLOW_COLOR, width=self._STROKE_GLOW_WIDTH, ) objects.append(glow_line) - line = staticmaps.Line( + line = _WebMercatorLine( points, color=stroke_color, width=width, @@ -1275,8 +1299,27 @@ def generate_map( hydrate_asset=hydrate_asset, ) + # Prefer best geometry only when every coordinate is inside the + # renderable Web Mercator latitude range. Staticmaps' center/zoom + # math raises at/near the poles, so polar records use the clamped + # bbox fallback below. + geojson_dict = self._geometry_to_geojson_dict(geometry) + if ( + geojson_dict is None + and bbox_coords[0] == bbox_coords[2] + and bbox_coords[1] == bbox_coords[3] + ): + geojson_dict = { + "type": "Point", + "coordinates": [bbox_coords[0], bbox_coords[1]], + } + map_objects = ( + self._geojson_to_staticmaps_objects(geojson_dict) + if geojson_dict and self._bbox_within_web_mercator(bbox_coords) + else None + ) render_bbox = self._renderable_bbox(bbox_coords) - if not render_bbox: + if not render_bbox and not map_objects: logger.debug( "Using global map for unrenderable Web-Mercator bbox on %s: %s", resource_id, @@ -1288,17 +1331,6 @@ def generate_map( hydrate_asset=hydrate_asset, ) - # Prefer best geometry only when every coordinate is inside the - # renderable Web Mercator latitude range. Staticmaps' center/zoom - # math raises at/near the poles, so polar records use the clamped - # bbox fallback below. - geojson_dict = self._geometry_to_geojson_dict(geometry) - map_objects = ( - self._geojson_to_staticmaps_objects(geojson_dict) - if geojson_dict and self._bbox_within_web_mercator(bbox_coords) - else None - ) - # Create a context for the map context = staticmaps.Context() context.set_tile_provider(tile_provider_Carto) @@ -1398,20 +1430,16 @@ def generate_basemap( hydrate_asset=hydrate_asset, ) - render_bbox = self._renderable_bbox(bbox_coords) - if not render_bbox: - logger.debug( - "Using global basemap for unrenderable Web-Mercator bbox on %s: %s", - resource_id, - bbox_coords, - ) - return self.generate_global_basemap( - resource_id, - source_signature=source_signature or self.geometry_signature(None), - hydrate_asset=hydrate_asset, - ) - geojson_dict = self._geometry_to_geojson_dict(geometry) + if ( + geojson_dict is None + and bbox_coords[0] == bbox_coords[2] + and bbox_coords[1] == bbox_coords[3] + ): + geojson_dict = { + "type": "Point", + "coordinates": [bbox_coords[0], bbox_coords[1]], + } extent_objects = ( self._geojson_to_staticmaps_objects( geojson_dict, @@ -1423,6 +1451,18 @@ def generate_basemap( if geojson_dict and self._bbox_within_web_mercator(bbox_coords) else None ) + render_bbox = self._renderable_bbox(bbox_coords) + if not render_bbox and not extent_objects: + logger.debug( + "Using global basemap for unrenderable Web-Mercator bbox on %s: %s", + resource_id, + bbox_coords, + ) + return self.generate_global_basemap( + resource_id, + source_signature=source_signature or self.geometry_signature(None), + hydrate_asset=hydrate_asset, + ) context = staticmaps.Context() context.set_tile_provider(tile_provider_Carto) diff --git a/backend/app/services/temporal_normalization.py b/backend/app/services/temporal_normalization.py new file mode 100644 index 0000000..7b0a3a3 --- /dev/null +++ b/backend/app/services/temporal_normalization.py @@ -0,0 +1,50 @@ +"""Helpers for normalizing OGM temporal indexing fields.""" + +from __future__ import annotations + +import re +from typing import Any, Optional + +_DATE_RANGE_START_YEAR = re.compile(r"^\s*\[?\s*(\d{1,4})(?=\D|$)") + + +def normalize_index_year(value: Any) -> Optional[list[int]]: + """Normalize an OGM index-year value to the database's ``list[int]`` shape.""" + values = value if isinstance(value, (list, tuple)) else [value] + years: list[int] = [] + + for candidate in values: + if isinstance(candidate, bool) or candidate is None: + continue + text = str(candidate).strip() + if text.isdigit(): + years.append(int(text)) + + return years or None + + +def derive_index_year(date_range: Any) -> Optional[list[int]]: + """Derive the start year from the first OGM date-range value. + + Bridge records store editable ranges as ``YYYY-YYYY`` while canonical + Aardvark records commonly use ``[YYYY TO YYYY]``. The legacy application + used the start year of the first range for ``gbl_indexYear_im``. + """ + if isinstance(date_range, (list, tuple)): + if not date_range: + return None + first_value = date_range[0] + else: + first_value = date_range + if first_value is None: + return None + + match = _DATE_RANGE_START_YEAR.match(str(first_value)) + if not match: + return None + return [int(match.group(1))] + + +def normalize_or_derive_index_year(index_year: Any, date_range: Any) -> Optional[list[int]]: + """Preserve valid supplied index years, otherwise derive the range's start year.""" + return normalize_index_year(index_year) or derive_index_year(date_range) diff --git a/backend/app/services/viewer_service.py b/backend/app/services/viewer_service.py index f5fa0a1..4b6569e 100644 --- a/backend/app/services/viewer_service.py +++ b/backend/app/services/viewer_service.py @@ -99,9 +99,10 @@ def parse_references( http_uri = "http://" + uri[len("https://") :] references.setdefault(http_uri, coerced) - # Add geometry if present + # Prefer the full geometry, but retain bbox-only records as viewer + # geometry. ItemViewer accepts both ENVELOPE and minx,miny,maxx,maxy. if isinstance(document, dict): - geom = document.get("locn_geometry") + geom = document.get("locn_geometry") or document.get("dcat_bbox") else: if hasattr(document, "locn_geometry"): geom = getattr(document, "locn_geometry", None) @@ -110,6 +111,12 @@ def parse_references( else: geom = None + if not geom: + if hasattr(document, "dcat_bbox"): + geom = getattr(document, "dcat_bbox", None) + elif hasattr(document, "get"): + geom = document.get("dcat_bbox", None) + if geom: references["locn_geometry"] = geom diff --git a/backend/app/viewers.py b/backend/app/viewers.py index 043bd9f..d4ddedd 100644 --- a/backend/app/viewers.py +++ b/backend/app/viewers.py @@ -162,6 +162,46 @@ def _parse_multipolygon_wkt(self, geometry: str) -> Optional[Dict[str, Union[str return None return {"type": "MultiPolygon", "coordinates": polygons} + @staticmethod + def _viewer_geometry_from_envelope( + minx: float, maxx: float, maxy: float, miny: float + ) -> Optional[GeoJSON]: + """Normalize an envelope and return consistently ordered viewer GeoJSON.""" + from app.elasticsearch.index import _normalize_envelope + + normalized_geom, error_msg = _normalize_envelope(minx, maxx, maxy, miny) + if normalized_geom is None: + logger.error( + "Invalid viewer envelope (%s, %s, %s, %s): %s - skipping", + minx, + maxx, + maxy, + miny, + error_msg, + ) + return None + + if normalized_geom["type"] != "polygon": + return {"type": "Point", "coordinates": normalized_geom["coordinates"]} + + ring = normalized_geom["coordinates"][0] + xs = [point[0] for point in ring] + ys = [point[1] for point in ring] + normalized_minx, normalized_maxx = min(xs), max(xs) + normalized_miny, normalized_maxy = min(ys), max(ys) + return { + "type": "Polygon", + "coordinates": [ + [ + [normalized_minx, normalized_maxy], + [normalized_minx, normalized_miny], + [normalized_maxx, normalized_miny], + [normalized_maxx, normalized_maxy], + [normalized_minx, normalized_maxy], + ] + ], + } + def viewer_geometry(self) -> Optional[GeoJSON]: """Convert locn_geometry to a GeoJSON object.""" if not self.references.get("locn_geometry"): @@ -203,41 +243,45 @@ def viewer_geometry(self) -> Optional[GeoJSON]: except ValueError: return None - # Import normalization function from elasticsearch module - from app.elasticsearch.index import _normalize_envelope + result = self._viewer_geometry_from_envelope(minx, maxx, maxy, miny) + if result is None: + return None - # Normalize and validate the envelope coordinates - normalized_geom, error_msg = _normalize_envelope(minx, maxx, maxy, miny) + # Cache the result for performance + self._geometry_cache[geometry] = result + return result - if normalized_geom is None: - logger.error(f"Invalid envelope in viewer {geometry}: {error_msg} - skipping") + point_inner = self._unwrap_wkt(geometry, "POINT") + if point_inner is not None: + try: + coordinates = [float(value) for value in point_inner.split()] + except ValueError: return None - # Return the normalized geometry with coordinate ordering - # adjusted to match expected test order for ENVELOPE polygons: - # [top-left, bottom-left, bottom-right, top-right, close] - if normalized_geom["type"] == "polygon": - ring = normalized_geom["coordinates"][0] - xs = [pt[0] for pt in ring] - ys = [pt[1] for pt in ring] - minx, maxx = min(xs), max(xs) - miny, maxy = min(ys), max(ys) - ordered_ring = [ - [minx, maxy], - [minx, miny], - [maxx, miny], - [maxx, maxy], - [minx, maxy], - ] - coords = [ordered_ring] - result = {"type": "Polygon", "coordinates": coords} - else: - result = {"type": "Point", "coordinates": normalized_geom["coordinates"]} + from app.elasticsearch.index import _is_valid_point - # Cache the result for performance + if len(coordinates) < 2 or not _is_valid_point(coordinates): + logger.warning(f"Invalid point coordinates in viewer: {geometry} - skipping") + return None + + result = {"type": "Point", "coordinates": coordinates[:2]} self._geometry_cache[geometry] = result return result + # The bridge stores dcat_bbox as minx,miny,maxx,maxy. Bbox-only + # records use this path after parse_references aliases it as geometry. + bbox_parts = [part.strip() for part in geometry.split(",")] + if len(bbox_parts) == 4: + try: + minx, miny, maxx, maxy = map(float, bbox_parts) + except ValueError: + pass + else: + result = self._viewer_geometry_from_envelope(minx, maxx, maxy, miny) + if result is not None: + self._geometry_cache[geometry] = result + return result + polygon_inner = self._unwrap_wkt(geometry, "POLYGON", layers=2) if polygon_inner is not None: result = self._parse_polygon_coords(polygon_inner, geometry) diff --git a/backend/tests/api/v1/test_facet_endpoint.py b/backend/tests/api/v1/test_facet_endpoint.py index f6aad44..3b56479 100644 --- a/backend/tests/api/v1/test_facet_endpoint.py +++ b/backend/tests/api/v1/test_facet_endpoint.py @@ -380,13 +380,16 @@ async def test_filter_context( mock_sanitize.side_effect = lambda x: x response = await async_client.get( - "/api/v1/search/facets/schema_provider_s?include_filters[dct_spatial_sm][]=Minnesota" + "/api/v1/search/facets/schema_provider_s?" + "include_filters[dct_spatial_sm][]=Minnesota&" + "include_filter_operator=and" ) assert response.status_code == 200 # Verify include_filters were passed to get_facet_values call_args = mock_get_facet.call_args assert call_args.kwargs["include_filters"] == {"dct_spatial_sm": ["Minnesota"]} + assert call_args.kwargs["include_filter_operator"] == "and" @patch("app.api.v1.endpoint_modules.search.get_facet_values") @patch("app.api.v1.endpoint_modules.search.process_facet_response") diff --git a/backend/tests/api/v1/test_search_endpoints.py b/backend/tests/api/v1/test_search_endpoints.py index 6f74e91..774904c 100644 --- a/backend/tests/api/v1/test_search_endpoints.py +++ b/backend/tests/api/v1/test_search_endpoints.py @@ -5,8 +5,9 @@ from fastapi.testclient import TestClient from httpx import AsyncClient from starlette.requests import Request +from starlette.responses import JSONResponse -from app.api.v1.endpoint_modules.search import _handle_search +from app.api.v1.endpoint_modules.search import _build_semantic_search_cache_key, _handle_search from app.main import app from app.services.resource_representation_cache import RESOURCE_SEARCH_RESULT_REPRESENTATION_PROFILE from tests.utils.route_helpers import route_paths @@ -1000,3 +1001,72 @@ def test_search_case_sensitivity(self): response = client.get(f"/api/v1/search?q={query}") # Should handle case variations gracefully assert response.status_code in [200, 500] # Allow database errors in test env + + +@pytest.mark.asyncio +async def test_handle_search_forwards_drilldown_filter_operator(): + request = _build_request(b"include_filter_operator=and") + search_mock = AsyncMock( + return_value={ + "data": [], + "meta": {"pages": {"total_count": 0, "total_pages": 0}}, + "queryTime": {}, + } + ) + + with patch("app.api.v1.endpoint_modules.search.SearchService.search", search_mock): + response = await _handle_search( + request, + { + "page": 1, + "per_page": 20, + "meta": True, + "include_filter_operator": "and", + }, + ) + + assert response.status_code == 200 + assert search_mock.await_args.kwargs["include_filter_operator"] == "and" + + +def test_semantic_search_cache_key_includes_filter_operator(): + common = { + "q": "", + "page": 1, + "per_page": 20, + "sort": None, + "search_field": None, + "fields": None, + "facets": None, + "include_filters": {"dct_spatial_sm": ["Indiana", "Indiana--Bloomington"]}, + "exclude_filters": {}, + "fq": {}, + "adv_q": None, + } + + or_key = _build_semantic_search_cache_key(**common, include_filter_operator="or") + and_key = _build_semantic_search_cache_key(**common, include_filter_operator="and") + + assert or_key != and_key + + +@pytest.mark.parametrize("method", ["get", "post"]) +@pytest.mark.parametrize("operator", ["and", "or"]) +def test_search_routes_forward_filter_operator(method, operator): + handler = AsyncMock(return_value=JSONResponse({"data": []})) + with patch("app.api.v1.endpoint_modules.search._handle_search", handler): + if method == "get": + response = client.get("/api/v1/search", params={"include_filter_operator": operator}) + else: + response = client.post("/api/v1/search", json={"include_filter_operator": operator}) + assert response.status_code == 200 + assert handler.await_args.args[1]["include_filter_operator"] == operator + + +@pytest.mark.parametrize("method,expected_status", [("get", 422), ("post", 400)]) +def test_search_routes_reject_invalid_filter_operator(method, expected_status): + if method == "get": + response = client.get("/api/v1/search", params={"include_filter_operator": "xor"}) + else: + response = client.post("/api/v1/search", json={"include_filter_operator": "xor"}) + assert response.status_code == expected_status diff --git a/backend/tests/elasticsearch/test_facet_functions.py b/backend/tests/elasticsearch/test_facet_functions.py index 4e93899..794b94b 100644 --- a/backend/tests/elasticsearch/test_facet_functions.py +++ b/backend/tests/elasticsearch/test_facet_functions.py @@ -463,13 +463,16 @@ async def test_with_include_filters(self, mock_es): facet_name="schema_provider_s", query=None, fq=None, - include_filters={"dct_spatial_sm": ["Minnesota"]}, + include_filters={"dct_spatial_sm": ["Indiana", "Indiana--Bloomington"]}, + include_filter_operator="and", exclude_filters=None, adv_q=None, ) - # Verify ES was called mock_es.search.assert_called_once() + filters = mock_es.search.call_args.kwargs["query"]["bool"]["filter"] + assert {"term": {"dct_spatial_sm.keyword": "Indiana"}} in filters + assert {"term": {"dct_spatial_sm.keyword": "Indiana--Bloomington"}} in filters @pytest.mark.asyncio @patch("app.elasticsearch.search.es") diff --git a/backend/tests/elasticsearch/test_index.py b/backend/tests/elasticsearch/test_index.py index 0e74cd7..36b2b7d 100644 --- a/backend/tests/elasticsearch/test_index.py +++ b/backend/tests/elasticsearch/test_index.py @@ -133,3 +133,30 @@ async def test_process_resource_does_not_fallback_to_tags_with_empty_explicit_og ) assert "ogm_repo" not in indexed + + +@pytest.mark.asyncio +async def test_process_resource_derives_index_year_from_date_range(monkeypatch): + async def fake_get_resource_summaries(resource_id): + return [] + + async def fake_get_spatial_facets(resource_id): + return None + + async def fake_get_allmaps_overlay_status(resource_id): + return False + + monkeypatch.setattr(index_module, "get_resource_summaries", fake_get_resource_summaries) + monkeypatch.setattr(index_module, "get_spatial_facets", fake_get_spatial_facets) + monkeypatch.setattr(index_module, "get_allmaps_overlay_status", fake_get_allmaps_overlay_status) + + indexed = await index_module.process_resource( + { + "id": "bridge-resource", + "gbl_indexYear_im": None, + "gbl_dateRange_drsim": ["2024-2024"], + } + ) + + assert indexed["gbl_indexYear_im"] == [2024] + assert indexed["time_period"] == "2020-2024" diff --git a/backend/tests/elasticsearch/test_search.py b/backend/tests/elasticsearch/test_search.py index f582225..1dfc2d2 100644 --- a/backend/tests/elasticsearch/test_search.py +++ b/backend/tests/elasticsearch/test_search.py @@ -11,6 +11,7 @@ BBOX_SPATIAL_BOOST_WEIGHT, MIN_BBOX_IOU_OVERLAP_RATIO, _build_bbox_overlap_filter, + _build_exact_filter_clauses, _compute_bbox_spatial_metrics, _escape_query_string_brackets, _normalize_geo_bbox_bounds, @@ -22,6 +23,51 @@ class TestElasticsearchSearch: + def test_repeated_include_values_preserve_api_or_default(self): + clauses = _build_exact_filter_clauses( + "dct_spatial_sm", + ["Indiana", "Indiana--Bloomington"], + "or", + ) + + assert clauses == [ + {"terms": {"dct_spatial_sm.keyword": ["Indiana", "Indiana--Bloomington"]}} + ] + + def test_repeated_include_values_build_drilldown_and_clauses(self): + clauses = _build_exact_filter_clauses( + "dct_spatial_sm", + ["Indiana", "Indiana--Bloomington"], + "and", + ) + + assert clauses == [ + {"term": {"dct_spatial_sm.keyword": "Indiana"}}, + {"term": {"dct_spatial_sm.keyword": "Indiana--Bloomington"}}, + ] + + @pytest.mark.asyncio + async def test_map_h3_aggregation_uses_drilldown_include_filters(self): + mock_es = AsyncMock() + mock_response = MagicMock() + mock_response.body = { + "aggregations": { + "h3_terms": {"buckets": []}, + "global_bucket_agg": {"doc_count": 0}, + } + } + mock_es.search.return_value = mock_response + + with patch("app.elasticsearch.search.es", mock_es): + await map_h3_aggregation( + include_filters={"dct_spatial_sm": ["Indiana", "Indiana--Bloomington"]}, + include_filter_operator="and", + ) + + filters = mock_es.search.await_args.kwargs["query"]["bool"]["filter"] + assert {"term": {"dct_spatial_sm.keyword": "Indiana"}} in filters + assert {"term": {"dct_spatial_sm.keyword": "Indiana--Bloomington"}} in filters + """Test cases for Elasticsearch search functionality.""" @pytest.fixture(autouse=True) diff --git a/backend/tests/services/test_cache_service.py b/backend/tests/services/test_cache_service.py index 7c13333..88c321b 100644 --- a/backend/tests/services/test_cache_service.py +++ b/backend/tests/services/test_cache_service.py @@ -14,6 +14,7 @@ cached_endpoint, invalidate_cache_with_prefix, ) +from app.services.response_cache_codec import COMPRESSION_PREFIX, decode_response_record class FakeRedisPipeline: @@ -360,6 +361,112 @@ async def test_set_record_persists_durable_response_with_tags(self): assert mock_store_durable.await_args.kwargs["namespace"] == "search_ns" assert set(mock_store_durable.await_args.kwargs["tags"]) == {"search", "resource:r1"} + @pytest.mark.asyncio + async def test_compressed_record_keeps_ttl_and_durable_payload(self): + record = {"schema": 2, "body_b64": "e30=" * 3000, "etag": 'W/"original"'} + fake_redis = FakeRedis() + with ( + patch("app.services.cache_service.ENDPOINT_CACHE", True), + patch("app.services.cache_service.CACHE_REDIS_COMPRESSION_ENABLED", True), + patch( + "app.services.cache_service.store_durable_api_response", new=AsyncMock() + ) as store, + patch("app.services.cache_service.get_durable_api_response", new=AsyncMock()) as get, + ): + service = CacheService() + service._redis_client = fake_redis + assert await service.set_record("key", record, 123, namespace="resource") + key, encoded, ttl, _ = fake_redis.set_calls[0] + assert (key, ttl) == ("key", 123) + assert encoded.startswith(COMPRESSION_PREFIX) + assert decode_response_record(encoded) == record + assert store.await_args.args == ("key", record) + fake_redis.get_value = encoded + # Turning writes off must not prevent reading existing compressed entries. + with patch("app.services.cache_service.CACHE_REDIS_COMPRESSION_ENABLED", False): + assert await service.get_record("key") == record + get.assert_not_awaited() + + @pytest.mark.asyncio + async def test_corrupt_compressed_record_recovers_from_durable_cache(self): + record = {"schema": 2, "body_b64": "e30=", "hard_exp": time.time() + 60} + fake_redis = FakeRedis(get_value=COMPRESSION_PREFIX + b"broken") + with ( + patch("app.services.cache_service.ENDPOINT_CACHE", True), + patch( + "app.services.cache_service.get_durable_api_response", + new=AsyncMock(return_value=(record, set(), "resource")), + ), + patch( + "app.services.cache_service.store_durable_api_response", new=AsyncMock() + ) as store, + ): + service = CacheService() + service._redis_client = fake_redis + assert await service.get_record("key") == record + assert decode_response_record(fake_redis.set_calls[0][1]) == record + store.assert_not_awaited() + + @pytest.mark.asyncio + async def test_compressed_http_hit_preserves_body_etag_and_conditional_response(self): + from fastapi import FastAPI, Request + from fastapi.responses import JSONResponse + from httpx import ASGITransport, AsyncClient + + class MemoryRedis(FakeRedis): + def __init__(self): + super().__init__() + self.values = {} + + async def get(self, key): + return self.values.get(key) + + async def set(self, key, value, ex=None, nx=False): + if nx and key in self.values: + return False + self.values[key] = value + return await super().set(key, value, ex=ex, nx=nx) + + redis = MemoryRedis() + with ( + patch("app.services.cache_service.ENDPOINT_CACHE", True), + patch("app.services.cache_service.CACHE_REDIS_COMPRESSION_ENABLED", True), + patch("app.services.cache_service.CACHE_DEBUG_HEADERS", True), + patch( + "app.services.cache_service.get_durable_api_response", + new=AsyncMock(return_value=None), + ), + patch("app.services.cache_service.store_durable_api_response", new=AsyncMock()), + ): + service = CacheService() + service._redis_client = redis + app = FastAPI() + calls = 0 + + @app.get("/compressed") + @cached_endpoint(ttl=60) + async def endpoint(request: Request): + nonlocal calls + calls += 1 + return JSONResponse({"data": [{"title": "Example resource"}] * 500}) + + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test" + ) as client: + first = await client.get("/compressed") + second = await client.get("/compressed") + conditional = await client.get( + "/compressed", headers={"If-None-Match": first.headers["etag"]} + ) + assert first.status_code == second.status_code == 200 + assert first.content == second.content + assert first.headers["etag"] == second.headers["etag"] + assert second.headers["x-cache"] == "HIT" + assert conditional.status_code == 304 + assert conditional.content == b"" + assert calls == 1 + assert any(value.startswith(COMPRESSION_PREFIX) for value in redis.values.values()) + @pytest.mark.asyncio async def test_invalidate_tags_deletes_durable_responses_even_without_redis(self): with ( diff --git a/backend/tests/services/test_image_service.py b/backend/tests/services/test_image_service.py index cec389d..fc3749a 100644 --- a/backend/tests/services/test_image_service.py +++ b/backend/tests/services/test_image_service.py @@ -385,8 +385,7 @@ def test_get_thumbnail_source_url_contentdm_transform(self): } result = service._get_thumbnail_source_url(references) assert ( - "cdm16022.contentdm.oclc.org/iiif/2/collection123:456/full/!800,800/0/default.jpg" - in result + "contentdm.oclc.org/iiif/2/collection123:456/full/!800,800/0/default.jpg" in result ) except Exception as e: @@ -1310,7 +1309,7 @@ def test_get_iiif_manifest_thumbnail_complex_sequences(self): result = service.get_iiif_manifest_thumbnail("http://example.com/manifest.json") # Should extract the image URL from the complex structure - assert result == "http://example.com/complex-image.jpg" + assert result == "http://example.com/iiif/service/full/!800,800/0/default.jpg" except Exception as e: # Handle Redis connection errors gracefully @@ -1702,8 +1701,7 @@ def test_contentdm_iiif_url_parsing(self): # Should transform to proper ContentDM IIIF format assert ( - "cdm16022.contentdm.oclc.org/iiif/2/collection123:456/full/!800,800/0/default.jpg" - in result + "contentdm.oclc.org/iiif/2/collection123:456/full/!800,800/0/default.jpg" in result ) except Exception as e: @@ -1774,3 +1772,120 @@ def test_wms_thumbnail_generation(self): except Exception as e: # Handle Redis connection errors gracefully assert _is_redis_connection_error(e) + + +class TestIssue412ThumbnailSources: + """Provider shapes observed in the collections reported in issue #412.""" + + @pytest.mark.parametrize( + "source", + [ + "https://cdm17287.contentdm.oclc.org/digital/iiif/wpamaps/2730/info.json", + "https://cdm17287.contentdm.oclc.org/iiif/2/wpamaps:2730/info.json", + "https://cdm17287.contentdm.oclc.org/iiif/2/wpamaps:2730/full/200,/0/default.jpg", + ], + ) + def test_pennsylvania_image_keeps_provider_and_identifier(self, source): + service = ImageService({}) + expected = "https://cdm17287.contentdm.oclc.org/iiif/2/wpamaps:2730/" + expected += "info.json" if source.endswith("info.json") else "full/!800,800/0/default.jpg" + assert service._get_thumbnail_source_url({"http://iiif.io/api/image": source}) == expected + + @pytest.mark.parametrize( + "path", + ["info/p16022coll231/10001", "p16022coll231:10001"], + ) + def test_contentdm_manifest_is_resolved_in_worker(self, path): + manifest_url = f"https://cdm16022.contentdm.oclc.org/iiif/{path}/manifest.json" + service = ImageService({}) + with patch.object(service, "_queue_thumbnail_processing") as queue: + assert ( + service._get_thumbnail_source_url( + {"http://iiif.io/api/presentation#manifest": manifest_url} + ) + == manifest_url + ) + queue.assert_not_called() + + @pytest.mark.parametrize("service_as_list", [False, True]) + def test_osu_uses_loris_service_instead_of_catalog_page(self, service_as_list): + image_service = {"@id": "https://library.osu.edu/loris/5h73q714r.jp2"} + resource = { + "@id": "https://library.osu.edu/dc/concern/file_sets/5h73q714r", + "service": [image_service] if service_as_list else image_service, + } + manifest = {"sequences": [{"canvases": [{"images": [{"resource": resource}]}]}]} + assert ImageService({})._extract_thumbnail_from_manifest_json(manifest) == ( + "https://library.osu.edu/loris/5h73q714r.jp2/full/!800,800/0/default.jpg" + ) + + def test_minnesota_compound_object_uses_first_page_service(self): + root = "https://cdm16022.contentdm.oclc.org/iiif" + manifest = { + "@id": f"{root}/p16022coll231:10001/manifest.json", + "sequences": [ + { + "canvases": [ + { + "images": [ + { + "resource": { + "@id": ( + f"{root}/2/p16022coll231:9957/full/full/0/default.jpg" + ), + "service": {"@id": f"{root}/2/p16022coll231:9957"}, + } + } + ] + } + ] + } + ], + } + assert ImageService({})._extract_thumbnail_from_manifest_json(manifest) == ( + f"{root}/2/p16022coll231:9957/full/!800,800/0/default.jpg" + ) + + def test_v2_direct_image_without_service_still_works(self): + image = "https://example.org/map.jpg" + manifest = { + "sequences": [ + { + "canvases": [ + { + "images": [ + { + "resource": { + "@id": image, + } + } + ] + } + ] + } + ] + } + assert ImageService({})._extract_thumbnail_from_manifest_json(manifest) == image + + def test_v3_service_without_iiif_path(self): + manifest = { + "items": [ + { + "items": [ + { + "items": [ + { + "body": { + "id": "https://example.org/catalog/map", + "service": [{"id": "https://example.org/loris/map.jp2"}], + } + } + ] + } + ] + } + ] + } + assert ImageService({})._extract_thumbnail_from_manifest_json(manifest) == ( + "https://example.org/loris/map.jp2/full/!800,800/0/default.jpg" + ) diff --git a/backend/tests/services/test_ogm_importer_normalization.py b/backend/tests/services/test_ogm_importer_normalization.py index 0717d8b..f976e9e 100644 --- a/backend/tests/services/test_ogm_importer_normalization.py +++ b/backend/tests/services/test_ogm_importer_normalization.py @@ -63,3 +63,28 @@ def test_normalize_record_injects_repo_tags(): normalized = importer._normalize_record(record, repo_name="edu.unr") assert normalized["b1g_adminTags_sm"] == ["curated", "ogm_repo:edu.unr", "ogm:unr"] + + +def test_normalize_record_derives_index_year_from_date_range(): + importer = OGMResourceImporter() + record = { + "id": "test-id", + "gbl_dateRange_drsim": ["[1922 TO 1962]"], + } + + normalized = importer._normalize_record(record, repo_name="edu.utexas") + + assert normalized["gbl_indexYear_im"] == [1922] + + +def test_normalize_record_preserves_supplied_index_years(): + importer = OGMResourceImporter() + record = { + "id": "test-id", + "gbl_indexYear_im": [1922, 1924, 1927], + "gbl_dateRange_drsim": ["[1922 TO 1962]"], + } + + normalized = importer._normalize_record(record, repo_name="edu.utexas") + + assert normalized["gbl_indexYear_im"] == [1922, 1924, 1927] diff --git a/backend/tests/services/test_response_cache_codec.py b/backend/tests/services/test_response_cache_codec.py new file mode 100644 index 0000000..a6e4307 --- /dev/null +++ b/backend/tests/services/test_response_cache_codec.py @@ -0,0 +1,66 @@ +import base64 +import json +import zlib + +import pytest + +from app.services import response_cache_codec as codec + + +def test_compressed_record_preserves_binary_body_and_metadata(): + record = { + "schema": 2, + "body_b64": base64.b64encode(bytes(range(256)) * 100).decode(), + "headers": {"content-type": "application/octet-stream"}, + "etag": 'W/"unchanged"', + "soft_exp": 100, + "hard_exp": 200, + "status": 200, + } + encoded = codec.encode_response_record(record, compress=True) + assert encoded.startswith(codec.COMPRESSION_PREFIX) + assert len(encoded) < len(json.dumps(record)) / 2 + assert codec.decode_response_record(encoded) == record + + +@pytest.mark.parametrize("as_text", [False, True]) +def test_reads_legacy_json(as_text): + record = {"schema": 2, "body_b64": "e30="} + raw = json.dumps(record) + assert codec.decode_response_record(raw if as_text else raw.encode()) == record + + +def test_compression_is_opt_in_and_small_records_stay_json(): + for record, enabled in [({"body": "x" * 10000}, False), ({"body": "small"}, True)]: + assert json.loads(codec.encode_response_record(record, compress=enabled)) == record + + +def test_skips_values_without_meaningful_savings(monkeypatch): + monkeypatch.setattr(codec, "MIN_COMPRESSION_BYTES", 1) + record = {"body": "abcdef"} + raw = codec.encode_response_record(record) + encoded = codec.encode_response_record(record, compress=True) + assert encoded == raw + assert codec.decode_response_record(encoded) == record + + +@pytest.mark.parametrize("kind", ["truncated", "trailing", "invalid", "unknown_version"]) +def test_rejects_invalid_compressed_values(kind): + encoded = codec.encode_response_record({"body": "x" * 10000}, compress=True) + values = { + "truncated": encoded[:-1], + "trailing": encoded + b"extra", + "invalid": codec.COMPRESSION_PREFIX + b"invalid zlib", + "unknown_version": b"OGM-RC\x02" + encoded[len(codec.COMPRESSION_PREFIX) :], + } + with pytest.raises((ValueError, zlib.error)): + codec.decode_response_record(values[kind]) + + +def test_bounds_decompression_and_leaves_oversized_records_uncompressed(monkeypatch): + record = {"body": "x" * 20000} + encoded = codec.encode_response_record(record, compress=True) + monkeypatch.setattr(codec, "MAX_DECOMPRESSED_BYTES", 10000) + with pytest.raises(ValueError, match="size limit"): + codec.decode_response_record(encoded) + assert json.loads(codec.encode_response_record(record, compress=True)) == record diff --git a/backend/tests/services/test_search_service.py b/backend/tests/services/test_search_service.py index a50b0b7..614258f 100644 --- a/backend/tests/services/test_search_service.py +++ b/backend/tests/services/test_search_service.py @@ -3,12 +3,31 @@ """ from unittest.mock import patch +from urllib.parse import urlencode import pytest from app.services.search_service import SearchService +def test_extract_new_style_filters_decodes_query_parameters_once(): + """Encoded reserved characters must remain part of facet values.""" + service = SearchService() + local_collection = "University of Maryland: U.S. Government Information, Maps, & GIS Services" + excluded_publisher = "C++ Maps & Data" + query_string = urlencode( + [ + ("include_filters[b1g_localCollectionLabel_sm][]", local_collection), + ("exclude_filters[dct_publisher_sm][]", excluded_publisher), + ] + ) + + include, exclude = service.extract_new_style_filters(query_string) + + assert include == {"b1g_localCollectionLabel_sm": [local_collection]} + assert exclude == {"dct_publisher_sm": [excluded_publisher]} + + @pytest.mark.asyncio async def test_search_preserves_search_payload_and_adds_lightweight_timings(): """SearchService should not enrich each hit when the endpoint rebuilds final resources.""" @@ -45,6 +64,22 @@ async def test_search_forwards_hydrate_hits_flag(): assert mock_search.call_args.kwargs["hydrate_hits"] is False +@pytest.mark.asyncio +async def test_search_forwards_include_filter_operator(): + service = SearchService() + + with patch("app.services.search_service.search_resources") as mock_search: + mock_search.return_value = {"data": [], "meta": {}, "queryTime": {}} + + await service.search( + q="", + include_filters={"dct_spatial_sm": ["Indiana", "Indiana--Bloomington"]}, + include_filter_operator="and", + ) + + assert mock_search.call_args.kwargs["include_filter_operator"] == "and" + + @pytest.mark.asyncio async def test_search_can_skip_result_sanitization_for_internal_callers(): service = SearchService() @@ -993,6 +1028,25 @@ def test_extract_new_style_filters_geo_distance(self): assert dist["center"] == {"lat": 43.5, "lon": -106.2} assert exclude == {} + def test_extract_new_style_filters_ignores_array_style_year_range(self): + """Malformed year arrays must not replace structured range bounds.""" + service = SearchService() + params = ( + "include_filters[year_range][start]=1920&" + "include_filters[year_range][end]=1929&" + "include_filters[year_range][]=1920&" + "include_filters[year_range][]=1929&" + "include_filters[dcat_theme_sm][]=Boundaries" + ) + + include, exclude = service.extract_new_style_filters(params) + + assert include == { + "year_range": {"start": "1920", "end": "1929"}, + "dcat_theme_sm": ["Boundaries"], + } + assert exclude == {} + @pytest.mark.asyncio async def test_search_passes_geospatial_include_filters(self): """Search should forward geo include_filters unchanged.""" diff --git a/backend/tests/services/test_static_map_service.py b/backend/tests/services/test_static_map_service.py index 7c3759e..1919cd6 100644 --- a/backend/tests/services/test_static_map_service.py +++ b/backend/tests/services/test_static_map_service.py @@ -1,6 +1,8 @@ import json from unittest.mock import MagicMock, call, patch +import staticmaps + from app.services.static_map_service import StaticMapService from app.services.visual_asset_cache import cache_visual_asset from tests.utils.distribution_helpers import make_distribution_context, make_distribution_record @@ -300,3 +302,93 @@ def test_generate_basemap_uses_global_fallback_for_unrenderable_polar_extent(): assert result == b"global-basemap" mock_global.assert_called_once() mock_render.assert_not_called() + + +def test_polygon_segments_are_not_replaced_with_geodesic_arcs(): + service = StaticMapService() + geometry = { + "type": "Polygon", + "coordinates": [[[-115, 55], [-65, 55], [-65, 30], [-115, 30], [-115, 55]]], + } + + objects = service._geojson_to_staticmaps_objects(geometry) + + assert objects is not None + assert len(objects[-1].interpolate()) == 6 + assert {round(point.lat().degrees, 6) for point in objects[-1].interpolate()} == { + 30.0, + 55.0, + } + + +def test_multipolygon_extent_does_not_take_short_path_across_antimeridian(): + service = StaticMapService() + geometry = { + "type": "MultiPolygon", + "coordinates": [ + [[[170, 20], [179, 20], [179, 30], [170, 30], [170, 20]]], + [[[-179, -30], [-170, -30], [-170, -20], [-179, -20], [-179, -30]]], + ], + } + + objects = service._geojson_to_staticmaps_objects(geometry) + + assert objects is not None + extent_points = objects[-1].interpolate() + assert len(extent_points) == 5 + assert [point.lng().degrees for point in extent_points] == [ + -179.0, + -179.0, + 179.0, + 179.0, + -179.0, + ] + + +def test_point_geometry_creates_visible_static_map_marker(): + service = StaticMapService() + + objects = service._geojson_to_staticmaps_objects( + {"type": "Point", "coordinates": [-87.62, 43.08]} + ) + + assert objects is not None + assert len(objects) == 1 + assert isinstance(objects[0], staticmaps.Marker) + assert objects[0].latlng().lng().degrees == -87.62 + assert objects[0].latlng().lat().degrees == 43.08 + assert objects[0].size() == 10 + + +def test_generate_map_renders_zero_area_bbox_as_point(): + service = StaticMapService() + + with ( + patch.object(service, "generate_global_map") as mock_global, + patch.object(service, "_render_and_cache", return_value=b"point-map") as mock_render, + ): + result = service.generate_map( + "bike-elevator", + "ENVELOPE(-87.62, -87.62, 43.08, 43.08)", + ) + + assert result == b"point-map" + mock_global.assert_not_called() + mock_render.assert_called_once() + + +def test_generate_basemap_uses_point_to_set_extent(): + service = StaticMapService() + + with ( + patch.object(service, "generate_global_basemap") as mock_global, + patch.object(service, "_render_and_cache", return_value=b"point-basemap") as mock_render, + ): + result = service.generate_basemap( + "bike-elevator", + "POINT(-87.62 43.08)", + ) + + assert result == b"point-basemap" + mock_global.assert_not_called() + mock_render.assert_called_once() diff --git a/backend/tests/services/test_temporal_normalization.py b/backend/tests/services/test_temporal_normalization.py new file mode 100644 index 0000000..1419c9d --- /dev/null +++ b/backend/tests/services/test_temporal_normalization.py @@ -0,0 +1,23 @@ +import pytest + +from app.services.temporal_normalization import normalize_or_derive_index_year + + +@pytest.mark.parametrize( + "supplied,ranges,expected", + [ + ([1922, 1924], ["[1900 TO 1950]"], [1922, 1924]), + (None, ["[1922 TO 1962]"], [1922]), + ([], ["2024-2024"], [2024]), + (None, ["[1922 TO 1962]", "[1800 TO 1850]"], [1922]), + ([True, None, "bad"], ["[1922 TO 1962]"], [1922]), + (" 1922 ", None, [1922]), + (None, [], None), + (None, ["unknown", "[1922 TO 1962]"], None), + (None, ["[* TO 1962]"], None), + ], +) +def test_temporal_fallback_preserves_explicit_years_and_uses_only_first_range( + supplied, ranges, expected +): + assert normalize_or_derive_index_year(supplied, ranges) == expected diff --git a/backend/tests/services/test_viewer_service.py b/backend/tests/services/test_viewer_service.py index a9af577..03e004f 100644 --- a/backend/tests/services/test_viewer_service.py +++ b/backend/tests/services/test_viewer_service.py @@ -82,6 +82,23 @@ def test_parse_references_with_dict_and_no_geometry(self): assert result["iiif"] == "http://example.com/iiif" assert "locn_geometry" not in result + def test_parse_references_uses_bbox_when_full_geometry_is_missing(self): + document = {"dcat_bbox": "-87.62,43.08,-87.62,43.08"} + + result = parse_references(document) + + assert result["locn_geometry"] == "-87.62,43.08,-87.62,43.08" + + def test_parse_references_prefers_full_geometry_over_bbox(self): + document = { + "locn_geometry": "POINT(-87.62 43.08)", + "dcat_bbox": "-88,42,-87,44", + } + + result = parse_references(document) + + assert result["locn_geometry"] == "POINT(-87.62 43.08)" + def test_parse_references_with_object_with_getitem(self): """Test parsing references from object with __getitem__ method.""" @@ -216,6 +233,14 @@ def get(self, key, default=None): class TestCreateViewerAttributes: """Test cases for create_viewer_attributes function.""" + def test_create_viewer_attributes_for_bbox_only_point(self): + result = create_viewer_attributes({"dcat_bbox": "-87.62,43.08,-87.62,43.08"}) + + assert result["ui_viewer_geometry"] == { + "type": "Point", + "coordinates": [-87.62, 43.08], + } + @patch("app.services.viewer_service.ItemViewer") def test_create_viewer_attributes_with_dict(self, mock_item_viewer): """Test creating viewer attributes with dict document.""" diff --git a/backend/tests/test_map_h3_api.py b/backend/tests/test_map_h3_api.py index ba0ab08..479ccde 100644 --- a/backend/tests/test_map_h3_api.py +++ b/backend/tests/test_map_h3_api.py @@ -47,6 +47,7 @@ def test_map_h3_returns_resolution_hexes_global_count(mock_agg): assert call_kw["q"] == "maps" assert call_kw["bbox"] == "-94,44,-92,46" assert call_kw["resolution"] == 5 + assert call_kw["include_filter_operator"] == "or" @patch("app.api.v1.endpoint_modules.map.map_h3_aggregation", new_callable=AsyncMock) @@ -93,3 +94,23 @@ def test_map_h3_forwards_adv_q(mock_agg): {"op": "AND", "f": "dct_title_s", "q": "water"}, {"op": "AND", "f": "dct_spatial_sm", "q": "Pennsylvania"}, ] + + +@patch("app.api.v1.endpoint_modules.map.map_h3_aggregation", new_callable=AsyncMock) +def test_map_h3_forwards_drilldown_filter_operator(mock_agg): + mock_agg.return_value = {"resolution": 5, "hexes": [], "globalCount": 7} + client = TestClient(_make_app()) + + response = client.get( + "/api/v1/map/h3", + params={ + "include_filter_operator": "and", + "include_filters[dct_spatial_sm][]": [ + "Indiana", + "Indiana--Bloomington", + ], + }, + ) + + assert response.status_code == 200 + assert mock_agg.call_args.kwargs["include_filter_operator"] == "and" diff --git a/backend/tests/viewers/test_viewers.py b/backend/tests/viewers/test_viewers.py index 0c55314..7ae9405 100644 --- a/backend/tests/viewers/test_viewers.py +++ b/backend/tests/viewers/test_viewers.py @@ -70,6 +70,30 @@ def test_viewer_geometry_with_geojson(): assert geometry["coordinates"] == [0, 0] +def test_viewer_geometry_with_wkt_point(): + viewer = ItemViewer({"locn_geometry": "POINT(-87.6200 43.0800)"}) + + geometry = viewer.viewer_geometry() + + assert geometry == {"type": "Point", "coordinates": [-87.62, 43.08]} + + +def test_viewer_geometry_with_zero_area_envelope(): + viewer = ItemViewer({"locn_geometry": "ENVELOPE(-87.62, -87.62, 43.08, 43.08)"}) + + geometry = viewer.viewer_geometry() + + assert geometry == {"type": "Point", "coordinates": [-87.62, 43.08]} + + +def test_viewer_geometry_with_zero_area_csv_bbox(): + viewer = ItemViewer({"locn_geometry": "-87.62,43.08,-87.62,43.08"}) + + geometry = viewer.viewer_geometry() + + assert geometry == {"type": "Point", "coordinates": [-87.62, 43.08]} + + def test_viewer_geometry_with_multipolygon_wkt(): """MultiPolygon WKT returns GeoJSON MultiPolygon (preserves type for dashed extent).""" references = { diff --git a/docs/upstream_ports_0.9.1.md b/docs/upstream_ports_0.9.1.md new file mode 100644 index 0000000..28370ec --- /dev/null +++ b/docs/upstream_ports_0.9.1.md @@ -0,0 +1,98 @@ +# Selected upstream fixes through 0.9.1 + +This change selectively adapts shared backend behavior from `geobtaa/api`. +It retains OGM harvesting, durable thumbnail coverage, identity, visibility +rules, deployment settings, and existing Boolean-query interpretation. + +## Search and temporal behavior + +Encoded filter values are decoded once, so a value such as `Maps & Atlases` +remains one value. Year-range parsing accepts only the supported `start` and +`end` keys and ignores malformed nested array keys. + +Search GET/POST, facet values, and H3 maps accept `include_filter_operator`: + +- `or` is the default and preserves existing repeated-value behavior. +- `and` requires every selected value within a field, including legacy `fq` + filters. Different fields remain conjunctive. Exclusions retain their behavior. +- GET rejects invalid operators with HTTP 422; POST search rejects them with + HTTP 400. POST also accepts uppercase AND/OR and normalizes them. + +For example, a POST search body can require both spatial labels: + +```json +{ + "include_filters": {"dct_spatial_sm": ["Indiana", "Indiana--Bloomington"]}, + "include_filter_operator": "and" +} +``` + +Clients should send the same operator to search, facet, and map requests. +Generated facet-apply links preserve it. Search and facet cache keys include +the operator, while H3 endpoint caching distinguishes query parameters. + +OGM ingestion and Elasticsearch indexing derive a missing or invalid +`gbl_indexYear_im` from the start year of the first `gbl_dateRange_drsim` +value. Valid explicit years take precedence. This does not expand a range +into every covered year. A reindex repairs existing search documents; +existing database records need re-ingestion or a separately reviewed backfill +to expose the derived value in resource metadata too. + +## Thumbnails, static maps, and viewers + +CONTENTdm normalization preserves the source provider and treats compound +manifests as manifests rather than guessing their image IDs. IIIF v2/v3 +parsing prefers declared image services, including service URLs without an +`/iiif/` path. OGM still resolves Image API `info.json` documents in the worker +to support Level 0 advertised sizes, and the thumbnail endpoint retains +ownership of job queueing. + +Static lines and areas connect projected vertices directly, preventing +geodesic interpolation artifacts. Points, zero-area point extents, and +bbox-only viewer records now receive geometry support. Static-map variants +advance to `static_map_v9` and `static_basemap_v7`; old durable variants are +not reused for new requests. The current basemap provider and OGM global +fallback artwork remain in place. + +Previously resolved thumbnails and recorded failures can outlive a code +change. Use the existing bounded cache-prime workflow to force regeneration +of affected resources and retry failures/placeholders as appropriate. See +[cache priming](cache_priming.md); this PR does not initiate any production +reindexing, cache invalidation, or regeneration. + +## Optional Redis response compression + +`CACHE_REDIS_COMPRESSION_ENABLED=false` is the default. When enabled, response +records of at least 4 KiB are compressed with zlib level 1 only if encoding +saves at least 10%. Encoding/decoding is bounded to 16 MiB of uncompressed +JSON. The format marker is `OGM-RC` followed by version byte 1. + +Readers accept both legacy JSON and compressed records, even when compressed +writes are disabled. Durable database records, HTTP bodies, ETags, cache +lifetimes, and conditional responses keep their existing semantics. Invalid +Redis records use the existing durable-cache fallback or become cache misses. + +Deploy compatible readers everywhere before enabling writes. Turning the +setting off stops new compressed writes but does not convert existing Redis +entries. Before rolling back to older readers, allow those entries to expire +or invalidate the affected response cache. Measure memory savings and CPU/ +latency on the OGM workload before enabling it broadly. + +## Provenance and scope + +| Upstream SHA | Adaptation | +| --- | --- | +| `eb89c10` | Encoded facet values and regression tests. | +| `131fa98` | Year-range parsing and regression tests. | +| `f6e6573` | Temporal helper, OGM ingestion, indexing; Bridge changes excluded. | +| `4389011` | IIIF resolution; preserves OGM Level 0 and queue ownership behavior. | +| `4ca8e5f` | Direct projected static geometry and cache variants. | +| `9ee88fe` | Backend point/bbox geometry support and regression tests. | +| `80dcd59` | AND faceting across endpoints, query builders, links, caches, and tests; visibility changes excluded. | +| `5b35609` | Opt-in response compression with OGM marker; host memory settings excluded. | + +The source review is in the [September review](upstream_review_2026-09-12.md). +The existing `app.identity` already centralizes API version reporting from +OGM package metadata, so upstream's separate release JSON system is not added. +Dependency refresh, grouped Boolean semantics, relationship visibility, +institutional facets, and tile-provider changes remain separate work. diff --git a/docs/upstream_reconciliation.md b/docs/upstream_reconciliation.md index 0a77267..c0cfbf5 100644 --- a/docs/upstream_reconciliation.md +++ b/docs/upstream_reconciliation.md @@ -99,3 +99,16 @@ pull requests and pushes to `develop`. Before the next upstream review, fetch `upstream`, choose a new immutable review ceiling, append every backend-affecting commit to this ledger, and keep earlier decisions intact for auditability. + +## September 12, 2026 review: 0.8.11 through 0.9.1 + +The next review ceiling is fetched upstream +`9a4c78419e371f83642b3b1330f9fd804d3e0ff7`. The +[September migration review](upstream_review_2026-09-12.md) extends this ledger +with all 33 backend-affecting non-merge commits since `4254aa3`, priority +recommendations, compatibility considerations, and proposed validation. +The original review records candidate decisions. The subsequent +[selected-port implementation](upstream_ports_0.9.1.md) applies `eb89c10`, +`131fa98`, `f6e6573`, `4389011`, `4ca8e5f`, `9ee88fe`, `80dcd59`, and +`5b35609` selectively, with OGM adaptations and regression coverage. +The full-import baseline and earlier port decisions remain unchanged. diff --git a/docs/upstream_review_2026-09-12.md b/docs/upstream_review_2026-09-12.md new file mode 100644 index 0000000..a6d4d19 --- /dev/null +++ b/docs/upstream_review_2026-09-12.md @@ -0,0 +1,147 @@ +# Upstream migration review — September 12, 2026 + +This document preserves the initial assessment. See the subsequent +[selected-port implementation](upstream_ports_0.9.1.md) for changes implemented +after the review. In particular, OGM's existing `app.identity` already +centralizes API version reporting, so a separate release JSON system was +not needed. + +Review OGM `7ba492f` against fetched `geobtaa/api` develop +`9a4c78419e371f83642b3b1330f9fd804d3e0ff7` (0.9.1). +The local data-api checkout at `e0dd1ac` contains the same reviewed backend; +the fetched ceiling adds the release-tag merge. The previous review ceiling +was `4254aa3` (0.8.11). There are 33 non-merge backend-affecting commits in +this interval. This is a source review, not an applied migration or runtime +validation. No application code, lockfile, or import baseline was changed. + +## Recommended first ports + +| Priority | Upstream change | Why it applies here | Port scope and acceptance | +| --- | --- | --- | --- | +| 1 | `eb89c10` encoded facet values; `131fa98` year-range parsing | OGM still unquotes the entire query string before `parse_qs`, which turns encoded ampersands into separators. Its broad year-range parsing also accepts malformed nested keys. | Small patches in `services/search_service.py` plus upstream regression cases. Verify ampersands, plus signs, encoded brackets, valid start/end bounds, and ignored nested year-range array keys. The complete frontend chip-removal fix is outside this repository. | +| 1 | `4389011` IIIF thumbnails | OGM still hardcodes one CONTENTdm host during URL conversion, guesses image IDs from manifest IDs, and prefers a v2 resource ID over a declared image service. Catalog-page IDs and compound objects can therefore yield invalid thumbnails. | Adapt manifest parsing and URL resolution with upstream CONTENTdm/OSU tests. Preserve OGM's extra info.json handling, durable thumbnail state, source-change invalidation, and coverage pipeline. Verify v2/v3 services, list-valued services, provider hostname preservation, compound manifests, and existing OGM tests. Repair affected cached resolution/failure state after the port; code changes alone may not repair existing assets. | +| 1 | `f6e6573` derive index year from date ranges | OGM currently normalizes supplied years but does not supply this fallback. Records with only `gbl_dateRange_drsim` miss expected year-facet entries. | Add the temporal helper and use it in OGM ingestion and Elasticsearch processing; Bridge is optional. Preserve explicit years. Test canonical `[1922 TO 1962]`, malformed/empty ranges, and multiple explicit years. Reindex to repair existing search documents; persisted resource metadata requires re-ingestion or a separate backfill. This derives the first range's start year, not every covered year. | +| 1 | `4ca8e5f` static geometry interpolation; `9ee88fe` point support | OGM retains static-map v7/basemap v5 behavior, geodesic interpolation, and incomplete point/bbox viewer handling. | Port together across static maps, viewer reference parsing, and ItemViewer. Carry geometry tests for direct projected lines, zero-area point extents, WKT points, bbox-only resources, and polar fallback. Advance OGM cache variants and verify durable assets regenerate. Retain OGM global fallback branding. These fixes do not require changing tile providers. | + +## Next candidates + +- **Drill-down faceting (`80dcd59`):** useful API addition across GET/POST + search, facet values, and H3 maps. Keep the existing OR default; clients + explicitly select AND. Port the parameter allowlists, validation, query + builders, service forwarding, links, and every affected cache key together. + Test that the same repeated selections produce consistent hits, counts, + and map totals, with separate AND/OR cache entries. Upstream context includes + the previously deferred visibility work; adapt hunks rather than importing + the whole search module. +- **Grouped Boolean search (`a0a69a2`):** replaces the current same-field OR + heuristic with ordered positive groups. Upstream interprets `A AND B OR C` + as `A AND (B OR C)`, with global NOT exclusions. This is an observable + compatibility change, not conventional Boolean precedence. Adopt only with + explicit API documentation and mixed-field, mixed-operator regression + coverage across search, facets, and maps; invalidate changed query caches. +- **Redis response compression (`5b35609`):** a good bounded performance + candidate for mirror nodes. Port only the codec, cache integration, tests, + and opt-in setting. Durable rows and HTTP bodies retain their format. + Upstream compresses sufficiently large records only when savings justify + it and bounds decompression. Choose an OGM format marker before enabling + writes. Deploy compatible readers before enabling compression; account for + old readers during rollback. Measure OGM memory savings and latency rather + than copying upstream institutional host allocations. +- **Dependency refresh (`1ebff73`, plus outstanding `9d2eb78`/`6f4b73b`):** + open a separate OGM dependency change. Current declared aiohttp, Pillow, + and Tornado pins are older than upstream; the MCP minimum is also lower. + Re-resolve the OGM lockfile and run a current vulnerability audit and full + suite. This review establishes version differences, not current advisory + status or that upstream versions are sufficient today. +- **Central version metadata (`f3c83a9`):** useful maintenance improvement: + one OGM-owned source for API root, OpenAPI, MCP, and package metadata. + Include package/build tests and ensure the release JSON is shipped in the + image/wheel. Do not copy upstream's version number or release automation. + +## Changes requiring an OGM-specific decision + +**Visibility remains unfinished prior work.** `7165142` was already marked +for selective porting. Resolve publication/suppression defaults and cache +isolation across public endpoints before importing broad upstream search +files. Decide whether any non-public diagnostic access belongs on the +restricted control plane rather than exposing an unrestricted public flag. + +`873d9c5` removes suppression filtering from upstream relationship widgets +while retaining publication filtering. OGM's current relationship query has +neither predicate, so this is not a missing one-line fix here. Define the +desired publication/suppression contract first, then apply it consistently +to relationship responses and representation caches. The older ledger's +description of a public relationship service should not be read as proof +that these predicates are currently enforced. + +`68c1f64` adds upstream institutional codes/admin tags to default full-text searches and supports +whole-day accession queries on upstream institutional fields. The shared query-builder +refactoring is reusable, but adding institutional administrative fields to +default public discovery is not an automatic OGM requirement. Likewise, +`a4006cd` adds a upstream institutional local-collection facet, not a general Aardvark collection +implementation. Defer unless corpus usage and client needs justify them. + +`bf95609` changes the basemap provider, attribution, user agent, zoom limit, +and cache variants. Evaluate a configurable provider for OGM separately, +including suitability for bulk cache generation. Preserve proper attribution +and use OGM identification if adopted. Do not couple this choice to the +independently useful geometry corrections. + +## Complete interval classification + +These are review decisions, not claims that ports have landed. + +| Commit | Decision | +| --- | --- | +| `19a869c` | Defer: Northwestern institutional access label. | +| `fc0bc0c` | Not applicable: backend release bookkeeping; named UI fixes are not backend implementations in this commit. | +| `a9f42ee` | Not applicable: backend release bookkeeping; UW viewer fix is not implemented in this backend delta. | +| `c6e00d3` | Optional small port: correct Waukesha fixture from array to object; validate local fixture ingestion. | +| `06a0cef` | Defer: Bridge cache rewarm rate limiting; retain as design input if an equivalent OGM rewarm issue is demonstrated. | +| `5a09ab2` | Not applicable: upstream institutional deployment memory settings and their tests. | +| `7d7fbdc` | Not applicable: release bookkeeping. | +| `f6e6573` | Port selectively: temporal fallback in OGM importer/indexer. | +| `a4006cd` | Defer: upstream institutional local collection facet. | +| `873d9c5` | Design decision: relationship visibility, with prior visibility work. | +| `4ca8e5f` | Port: direct projected static geometry. | +| `9ee88fe` | Port selectively: backend point and bbox viewer support; frontend work excluded. | +| `fb666ea` | Not applicable: release bookkeeping. | +| `80dcd59` | Port selectively: opt-in AND facet semantics and complete cache/endpoint propagation. | +| `a0a69a2` | Port selectively after API semantics decision: grouped Boolean queries. | +| `131fa98` | Port selectively: backend year-range parser and tests. | +| `db767c7` | Not applicable: release bookkeeping. | +| `eb89c10` | Port: parse encoded filters once. | +| `f3c83a9` | Port selectively: centralized OGM version metadata. | +| `d358141` | Not applicable: release bookkeeping. | +| `6f51ef0` | Not applicable: release metadata. | +| `9436630` | Defer: backup retention is an OGM operations decision. | +| `09411d7` | Not applicable: release metadata. | +| `bf95609` | Design decision: tile provider and attribution. | +| `e19ef34` | Not applicable: release metadata. | +| `74418ac` | Optional test port only: backend icon-gradient endpoint assertion; runtime fix is frontend. | +| `5d1dde2` | Not applicable: release metadata. | +| `5b35609` | Port selectively: opt-in response codec; exclude production memory configuration. | +| `1ebff73` | Dependency review: independently resolve and audit OGM dependencies. | +| `68c1f64` | Defer institutional search fields; shared query helper is reusable. | +| `b4ac94f` | Not applicable: release metadata. | +| `4389011` | Port selectively: IIIF resolution preserving OGM thumbnail extensions. | +| `36e25ab` | Not applicable: release metadata. | + +The latest last-facet reset fix (`dff2dca`, #415) is frontend-only and is +deliberately outside the backend ledger. It would belong in a consuming +discovery client with the equivalent URL/query-state behavior. + +## Suggested delivery order + +1. Small filter-parser corrections. +2. Temporal normalization with an explicit existing-record repair plan. +3. IIIF resolution adapted to OGM thumbnail state. +4. Static-map and viewer geometry fixes with cache version changes. +5. Visibility contract, then broader facet/Boolean capabilities. +6. Separately: dependency refresh, optional compression, and version metadata. + +Carry each port's focused upstream tests and OGM regressions, followed by +the complete backend suite before release. Keep the June full-import baseline +unchanged and record actual port SHAs in the reconciliation ledger when +implemented. The previously deferred distribution/asset work (`ab214d5`) +also remains outstanding; it was not superseded by this interval.