From 54c11829a85522180c2e1dfaac3b59ae94cbb8f6 Mon Sep 17 00:00:00 2001 From: Roger Abelenda <3179183+rabelenda@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:09:11 -0300 Subject: [PATCH] feat(glpi): add GLPI plugin and dashboard Collect GLPI entities, tickets, and related users and groups into the ticket domain, with MySQL and PostgreSQL Grafana dashboards. A full sync applies the project timeAfter already forwarded by the remote runner. PyDevLake reuses an existing raw-table model so a later subtask can read rows collected earlier in the same process. The connection form sends the GLPI server timezone, which the plugin requires. Refs #9273 Signed-off-by: Roger Abelenda <3179183+rabelenda@users.noreply.github.com> --- .github/dependabot.yml | 1 + backend/python/plugins/glpi/README.md | 54 + backend/python/plugins/glpi/build.sh | 20 + backend/python/plugins/glpi/glpi/__init__.py | 14 + backend/python/plugins/glpi/glpi/api.py | 257 ++++ .../python/plugins/glpi/glpi/collection.py | 88 ++ backend/python/plugins/glpi/glpi/main.py | 119 ++ .../python/plugins/glpi/glpi/migrations.py | 205 +++ backend/python/plugins/glpi/glpi/models.py | 242 +++ backend/python/plugins/glpi/glpi/parsing.py | 62 + .../plugins/glpi/glpi/streams/__init__.py | 14 + .../plugins/glpi/glpi/streams/groups.py | 80 + .../glpi/glpi/streams/ticket_domain.py | 166 +++ .../glpi/glpi/streams/ticket_substreams.py | 471 ++++++ .../plugins/glpi/glpi/streams/tickets.py | 282 ++++ .../python/plugins/glpi/glpi/streams/users.py | 82 ++ backend/python/plugins/glpi/glpi/timezone.py | 51 + backend/python/plugins/glpi/pyproject.toml | 37 + backend/python/plugins/glpi/run.sh | 20 + backend/python/plugins/glpi/tests/__init__.py | 14 + backend/python/plugins/glpi/tests/api_test.py | 191 +++ .../plugins/glpi/tests/collection_test.py | 225 +++ backend/python/plugins/glpi/tests/common.py | 72 + .../plugins/glpi/tests/enrichment_test.py | 313 ++++ .../python/plugins/glpi/tests/parsing_test.py | 49 + .../python/plugins/glpi/tests/plugin_test.py | 45 + .../plugins/glpi/tests/remote_scopes_test.py | 52 + .../python/plugins/glpi/tests/streams_test.py | 463 ++++++ .../plugins/glpi/tests/timezone_test.py | 21 + backend/python/pydevlake/pydevlake/stream.py | 28 +- .../connection-form/fields/index.tsx | 3 + .../connection-form/fields/timezone.tsx | 91 ++ .../components/connection-form/index.tsx | 2 + .../components/scope-config-form/index.tsx | 9 + .../src/plugins/register/glpi/assets/icon.svg | 24 + .../src/plugins/register/glpi/config.tsx | 82 ++ .../src/plugins/register/glpi/data-scope.tsx | 175 +++ config-ui/src/plugins/register/glpi/index.ts | 20 + .../plugins/register/glpi/transformation.tsx | 138 ++ config-ui/src/plugins/register/index.ts | 2 + config-ui/src/utils/timezones.ts | 100 ++ grafana/dashboards/mysql/Glpi.json | 1241 ++++++++++++++++ grafana/dashboards/postgresql/Glpi.json | 1301 +++++++++++++++++ 43 files changed, 6919 insertions(+), 7 deletions(-) create mode 100644 backend/python/plugins/glpi/README.md create mode 100755 backend/python/plugins/glpi/build.sh create mode 100644 backend/python/plugins/glpi/glpi/__init__.py create mode 100644 backend/python/plugins/glpi/glpi/api.py create mode 100644 backend/python/plugins/glpi/glpi/collection.py create mode 100644 backend/python/plugins/glpi/glpi/main.py create mode 100644 backend/python/plugins/glpi/glpi/migrations.py create mode 100644 backend/python/plugins/glpi/glpi/models.py create mode 100644 backend/python/plugins/glpi/glpi/parsing.py create mode 100644 backend/python/plugins/glpi/glpi/streams/__init__.py create mode 100644 backend/python/plugins/glpi/glpi/streams/groups.py create mode 100644 backend/python/plugins/glpi/glpi/streams/ticket_domain.py create mode 100644 backend/python/plugins/glpi/glpi/streams/ticket_substreams.py create mode 100644 backend/python/plugins/glpi/glpi/streams/tickets.py create mode 100644 backend/python/plugins/glpi/glpi/streams/users.py create mode 100644 backend/python/plugins/glpi/glpi/timezone.py create mode 100644 backend/python/plugins/glpi/pyproject.toml create mode 100755 backend/python/plugins/glpi/run.sh create mode 100644 backend/python/plugins/glpi/tests/__init__.py create mode 100644 backend/python/plugins/glpi/tests/api_test.py create mode 100644 backend/python/plugins/glpi/tests/collection_test.py create mode 100644 backend/python/plugins/glpi/tests/common.py create mode 100644 backend/python/plugins/glpi/tests/enrichment_test.py create mode 100644 backend/python/plugins/glpi/tests/parsing_test.py create mode 100644 backend/python/plugins/glpi/tests/plugin_test.py create mode 100644 backend/python/plugins/glpi/tests/remote_scopes_test.py create mode 100644 backend/python/plugins/glpi/tests/streams_test.py create mode 100644 backend/python/plugins/glpi/tests/timezone_test.py create mode 100644 config-ui/src/plugins/components/connection-form/fields/timezone.tsx create mode 100644 config-ui/src/plugins/register/glpi/assets/icon.svg create mode 100644 config-ui/src/plugins/register/glpi/config.tsx create mode 100644 config-ui/src/plugins/register/glpi/data-scope.tsx create mode 100644 config-ui/src/plugins/register/glpi/index.ts create mode 100644 config-ui/src/plugins/register/glpi/transformation.tsx create mode 100644 config-ui/src/utils/timezones.ts create mode 100644 grafana/dashboards/mysql/Glpi.json create mode 100644 grafana/dashboards/postgresql/Glpi.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 81ff1043350..c8ec178fbd1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -120,6 +120,7 @@ updates: - /backend/python - /backend/python/pydevlake - /backend/python/plugins/azuredevops + - /backend/python/plugins/glpi - /backend/python/plugins/redmine - /backend/python/test/fakeplugin - /grafana/scripts diff --git a/backend/python/plugins/glpi/README.md b/backend/python/plugins/glpi/README.md new file mode 100644 index 00000000000..b82cfea01c8 --- /dev/null +++ b/backend/python/plugins/glpi/README.md @@ -0,0 +1,54 @@ + +# GLPI + +Apache DevLake plugin for ingesting data from [GLPI](https://glpi-project.org/) through its REST API. Each DevLake scope is a GLPI entity. + +Tickets are normalized into DevLake's ticket domain. Users and groups are normalized into cross-domain accounts. + +## Collected data + +| GLPI resource | Raw table | Tool table | Domain tables | +| --- | --- | --- | --- | +| Tickets | `_raw_glpi_tickets` | `_tool_glpi_tickets` | `issues`, `board_issues`, `issue_labels` | +| Ticket users (optional) | `_raw_glpi_ticketusers` | `_tool_glpi_ticketusers` | `issue_assignees` | +| Ticket groups (optional) | `_raw_glpi_ticketgroups` | `_tool_glpi_ticketgroups` | `issue_assignees` | +| Follow-ups (optional) | `_raw_glpi_ticketfollowups` | `_tool_glpi_followups` | `issue_comments` | +| Tasks (optional) | `_raw_glpi_tickettasks` | `_tool_glpi_tickettasks` | `issue_worklogs` | +| History logs (optional) | `_raw_glpi_ticketlogs` | `_tool_glpi_ticketlogs` | `issue_changelogs` | +| Ticket links (optional) | `_raw_glpi_ticketlinks` | `_tool_glpi_ticketlinks` | `issue_relationships` | +| Solutions (optional) | `_raw_glpi_ticketsolutions` | `_tool_glpi_itilsolutions` | — | +| Users | `_raw_glpi_users` | `_tool_glpi_users` | `accounts` | +| Groups | `_raw_glpi_groups` | `_tool_glpi_groups` | `accounts` | +| Entities (scope) | — | `_tool_glpi_entities` | `boards` | + +Categories, locations, suppliers, request types, labels, and custom fields collected with tickets stay on `_tool_glpi_itilcategories`, `_tool_glpi_locations`, `_tool_glpi_suppliers`, `_tool_glpi_requesttypes`, `_tool_glpi_ticketlabelrecords`, and `_tool_glpi_ticketcustomfields`. + +A full sync applies the project sync policy `timeAfter` as a ticket creation filter. An incremental sync uses the last `date_mod`. + +## Configuration + +1. Enable the API under **Setup → General → API** and copy the application token. +2. Create a user API token under **My settings → Remote access keys**. +3. In Config UI, add a GLPI connection with the instance base URL, both tokens, and the timezone configured on the GLPI server. +4. Select entities. Assignees are collected unless disabled. Follow-ups, tasks, history, solutions, and ticket links stay off unless enabled on the scope config. + +Incident tickets map to `INCIDENT`. Requests map to `TASK`. New maps to `TODO`. Processing and Pending map to `IN_PROGRESS`. Solved and Closed map to `DONE`. Any other status maps to `OTHER`. + +## Dashboard + +After a blueprint run, open the GLPI dashboard in Grafana. MySQL and PostgreSQL definitions are in `grafana/dashboards/mysql/Glpi.json` and `grafana/dashboards/postgresql/Glpi.json`. diff --git a/backend/python/plugins/glpi/build.sh b/backend/python/plugins/glpi/build.sh new file mode 100755 index 00000000000..43613c9878a --- /dev/null +++ b/backend/python/plugins/glpi/build.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "$(dirname "$0")" +sh ../../uv.sh sync . diff --git a/backend/python/plugins/glpi/glpi/__init__.py b/backend/python/plugins/glpi/glpi/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/plugins/glpi/glpi/api.py b/backend/python/plugins/glpi/glpi/api.py new file mode 100644 index 00000000000..87f79ef7551 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/api.py @@ -0,0 +1,257 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Optional + +import requests as req + +from pydevlake.api import API, APIException, Paginator, Request, request_hook +from pydevlake.logger import logger + + +_API_PATH = 'apirest.php' +_PAGE_SIZE = 100 + +# GLPI Ticket table columns used for list sorting (not search-option IDs) +_TICKET_SORT_OPENING_DATE = 'date' +_TICKET_SORT_DATE_MOD = 'date_mod' + + +class GlpiPaginator(Paginator): + def get_items(self, response) -> list: + return response.json + + def get_next_page_id(self, response) -> Optional[int]: + content_range = response.headers.get('Content-Range', '') + match = re.match(r'(\d+)-(\d+)/(\d+)', content_range) + if not match: + return None + end, total = int(match.group(2)), int(match.group(3)) + next_start = end + 1 + return next_start if next_start < total else None + + def set_next_page_param(self, request, next_page_id): + request.query_args['range'] = f'{next_page_id}-{next_page_id + _PAGE_SIZE - 1}' + + +class GlpiAPI(API): + paginator = GlpiPaginator() + + def __init__(self, connection): + super().__init__(connection) + self._session_token = None + + @property + def base_url(self) -> str: + return str(self.connection.endpoint).rstrip('/') + + def test_connection(self): + self._init_session() + + def _init_session(self): + resp = req.get( + f'{self.base_url}/{_API_PATH}/initSession', + headers={ + 'App-Token': self.connection.token.get_secret_value(), + 'Authorization': f'user_token {self.connection.secret_key.get_secret_value()}', + } + ) + resp.raise_for_status() + self._session_token = resp.json()['session_token'] + + @request_hook + def authenticate(self, request: Request): + if self._session_token is None: + self._init_session() + request.headers['App-Token'] = self.connection.token.get_secret_value() + request.headers['Session-Token'] = self._session_token + + def __enter__(self): + return self + + def __exit__(self, *_): + if self._session_token is None: + return + try: + self.get(_API_PATH, 'killSession') + except APIException: + logger.warning("Failed to kill GLPI session", exc_info=True) + finally: + self._session_token = None + + def entities(self, expand_dropdowns='true'): + return self._api_get('Entity', expand_dropdowns=expand_dropdowns) + + def _api_get(self, resource, *path_args, expand_dropdowns='false', get_hateoas='false', **kwargs): + return self.get( + _API_PATH, resource, *path_args, + range=f'0-{_PAGE_SIZE - 1}', + expand_dropdowns=expand_dropdowns, + get_hateoas=get_hateoas, + **kwargs, + ) + + def iter_tickets( + self, + entity_id: Optional[str] = None, + updated_since: Optional[str] = None, + created_after: Optional[str] = None, + limit: Optional[int] = None, + ): + if updated_since: + yield from self._iter_ticket_list( + entity_id=entity_id, + limit=limit, + stop_before_date_mod=updated_since, + sort=_TICKET_SORT_DATE_MOD, + ) + return + if created_after: + yield from self._iter_ticket_list( + entity_id=entity_id, + limit=limit, + stop_before_opening_date=created_after, + sort=_TICKET_SORT_OPENING_DATE, + ) + return + yield from self._iter_ticket_list(entity_id=entity_id, limit=limit) + + @staticmethod + def _should_stop_ticket_page( + ticket: dict, + stop_before_date_mod: Optional[str], + stop_before_opening_date: Optional[str], + ) -> bool: + if stop_before_date_mod: + date_mod = ticket.get('date_mod') + if date_mod and date_mod <= stop_before_date_mod: + return True + if stop_before_opening_date: + opening_date = ticket.get('date') + if opening_date and opening_date <= stop_before_opening_date: + return True + return False + + def _ticket_list_query( + self, + entity_id: Optional[str], + sort: Optional[str] = None, + ) -> dict: + query = dict(expand_dropdowns='true', get_hateoas='false') + if sort is not None: + query['sort'] = sort + query['order'] = 'DESC' + if entity_id and entity_id != '0': + query['searchText[entities_id]'] = entity_id + return query + + def _iter_ticket_list( + self, + entity_id: Optional[str] = None, + limit: Optional[int] = None, + stop_before_date_mod: Optional[str] = None, + stop_before_opening_date: Optional[str] = None, + sort: Optional[str] = None, + ): + query = self._ticket_list_query(entity_id, sort) + has_date_stop = stop_before_date_mod or stop_before_opening_date + + if limit is None and not has_date_stop: + yield from self.get( + _API_PATH, 'Ticket', + range=f'0-{_PAGE_SIZE - 1}', + **query, + ) + return + + yielded = 0 + range_start = 0 + while limit is None or yielded < limit: + page_size = _PAGE_SIZE if limit is None else min(_PAGE_SIZE, limit - yielded) + response = self.get( + _API_PATH, 'Ticket', + range=f'{range_start}-{range_start + page_size - 1}', + **query, + ) + items = self.paginator.get_items(response) + if not items: + break + if not isinstance(items, list): + items = [items] + for item in items: + if self._should_stop_ticket_page( + item, stop_before_date_mod, stop_before_opening_date, + ): + return + yield item + yielded += 1 + if limit is not None and yielded >= limit: + return + + content_range = response.headers.get('Content-Range', '') + match = re.match(r'(\d+)-(\d+)/(\d+)', content_range) + if not match: + break + end, total = int(match.group(2)), int(match.group(3)) + if end + 1 >= total: + break + range_start = end + 1 + + @staticmethod + def _item_body(body) -> dict: + if isinstance(body, dict): + return body + if isinstance(body, list): + for entry in body: + if isinstance(entry, dict): + return entry + return {} + + def get_item(self, itemtype: str, item_id: int) -> dict: + if item_id <= 0: + return {} + try: + response = self.get( + _API_PATH, itemtype, str(item_id), + expand_dropdowns='false', get_hateoas='false', + ) + return self._item_body(response.json) + except APIException as exc: + if self._is_item_not_found(exc): + return {} + raise + + @staticmethod + def _is_item_not_found(exc: APIException) -> bool: + response = exc.response + if response.status != 404: + return False + body = response.body + if isinstance(body, bytes): + body = body.decode('utf-8', errors='replace') + text = str(body) + return 'ERROR_ITEM_NOT_FOUND' in text or 'Item not found' in text + + def iter_items(self, itemtype: str, expand_dropdowns='true'): + return self._api_get(itemtype, expand_dropdowns=expand_dropdowns) + + def ticket_subitems(self, ticket_id: int, sub_type: str): + return self._api_get( + 'Ticket', str(ticket_id), sub_type, expand_dropdowns='false', + ) + + def iter_ticket_links(self): + yield from self.iter_items('Ticket_Ticket', expand_dropdowns='false') diff --git a/backend/python/plugins/glpi/glpi/collection.py b/backend/python/plugins/glpi/glpi/collection.py new file mode 100644 index 00000000000..a5ff111c326 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/collection.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import Optional, Union + +from pydevlake.model import raw_data_params +from sqlmodel import Session + +from glpi.parsing import parse_glpi_dropdown + +_USER_RAW_STREAMS = ( + 'tickets', + 'ticketusers', + 'ticketfollowups', + 'tickettasks', + 'ticketlogs', +) + + +def scan_raw_rows(session: Session, plugin, stream_name: str, params: str): + stream = plugin.get_stream(stream_name) + raw_model = stream.raw_model(session) + return session.query(raw_model).where(raw_model.params == params).yield_per(500) + + +def prepare_on_demand_raw( + raw: dict, + item_id: int, + resource: str, + base_url: str, +) -> Optional[dict]: + if item_id <= 0 or not raw: + return None + raw.setdefault('id', item_id) + raw['x_request_url'] = f'{base_url}/apirest.php/{resource}/{item_id}' + return raw + + +def referenced_user_ids(session: Session, plugin, ctx) -> set[int]: + user_ids: set[int] = set() + params = raw_data_params(ctx.connection.id, ctx.scope.id) + for stream_name in _USER_RAW_STREAMS: + for raw in scan_raw_rows(session, plugin, stream_name, params): + data = json.loads(raw.data) + if stream_name == 'tickets': + _add_user_id(user_ids, data.get('users_id_recipient')) + _add_user_id(user_ids, data.get('users_id_lastupdater')) + elif stream_name == 'ticketusers': + _add_user_id(user_ids, data.get('users_id')) + elif stream_name == 'ticketfollowups': + _add_user_id(user_ids, data.get('users_id')) + elif stream_name == 'tickettasks': + _add_user_id(user_ids, data.get('users_id')) + _add_user_id(user_ids, data.get('users_id_tech')) + elif stream_name == 'ticketlogs': + _add_user_id(user_ids, data.get('user_name')) + _add_user_id(user_ids, data.get('users_id')) + return user_ids + + +def referenced_group_ids(session: Session, plugin, ctx) -> set[int]: + group_ids: set[int] = set() + params = raw_data_params(ctx.connection.id, ctx.scope.id) + for raw in scan_raw_rows(session, plugin, 'ticketgroups', params): + data = json.loads(raw.data) + _, group_id = parse_glpi_dropdown(data.get('groups_id')) + if group_id is not None: + group_ids.add(group_id) + return group_ids + + +def _add_user_id(user_ids: set[int], value: Optional[Union[str, int]]) -> None: + _, user_id = parse_glpi_dropdown(value) + if user_id is not None: + user_ids.add(int(user_id)) diff --git a/backend/python/plugins/glpi/glpi/main.py b/backend/python/plugins/glpi/glpi/main.py new file mode 100644 index 00000000000..1aa57275477 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/main.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable + +from glpi.api import GlpiAPI +from glpi.models import Connection, Entity, ScopeConfig +from glpi.timezone import validate_timezone_name +from glpi.streams.groups import Groups +from glpi.streams.ticket_domain import TicketDomain +from glpi.streams.ticket_substreams import ( + TicketFollowups, + TicketGroups, + TicketLinks, + TicketLogs, + TicketSolutions, + TicketTasks, + TicketUsers, +) +from glpi.streams.tickets import Tickets +from glpi.streams.users import Users + +from pydevlake import Plugin, RemoteScopeGroup, TestConnectionResult +from pydevlake.logger import logger +from pydevlake.model import raw_data_params +import pydevlake.domain_layer.ticket as ticket + +ROOT_GROUP_ID = 'root' + + +class GlpiPlugin(Plugin): + + @property + def connection_type(self): + return Connection + + @property + def tool_scope_type(self): + return Entity + + @property + def scope_config_type(self): + return ScopeConfig + + def domain_scopes(self, entity: Entity): + yield ticket.Board( + name=entity.completename or entity.name, + description=entity.completename, + type='ITSM', + ) + + def remote_scope_groups(self, connection) -> list[RemoteScopeGroup]: + return [RemoteScopeGroup(id=ROOT_GROUP_ID, name='Entities')] + + def remote_scopes(self, connection, group_id: str) -> list[Entity]: + with GlpiAPI(connection) as api: + yield from self._entities(api, connection) + + def _entities(self, api: GlpiAPI, connection: Connection) -> Iterable[Entity]: + raw_table = self._raw_scope_table_name() + for raw_entity in sorted( + api.entities(expand_dropdowns='false'), + key=lambda raw: (raw.get('completename') or raw.get('name') or str(raw['id'])).lower(), + ): + entity_id = str(raw_entity['id']) + entity = Entity( + id=entity_id, + name=raw_entity.get('name', ''), + completename=raw_entity.get('completename', raw_entity.get('name', '')), + ) + entity.connection_id = connection.id + entity.raw_data_params = raw_data_params(connection.id, entity_id) + entity.raw_data_table = raw_table + yield entity + + def test_connection(self, connection: Connection) -> TestConnectionResult: + try: + validate_timezone_name(connection.timezone) + except ValueError as e: + return TestConnectionResult(success=False, message=str(e), status=400) + with GlpiAPI(connection) as api: + try: + api.test_connection() + return TestConnectionResult(success=True, message='Connection successful', status=200) + except Exception as e: + logger.warning("GLPI test_connection failed", exc_info=True) + return TestConnectionResult(success=False, message=str(e), status=400) + + @property + def streams(self): + return [ + Tickets, + TicketUsers, + TicketGroups, + TicketFollowups, + TicketTasks, + TicketLogs, + TicketLinks, + TicketSolutions, + TicketDomain, + Users, + Groups, + ] + + +if __name__ == '__main__': + GlpiPlugin.start() diff --git a/backend/python/plugins/glpi/glpi/migrations.py b/backend/python/plugins/glpi/glpi/migrations.py new file mode 100644 index 00000000000..5313b4f30e5 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/migrations.py @@ -0,0 +1,205 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from typing import Optional + +from pydantic import AnyUrl, SecretStr + +from pydevlake import Field +from pydevlake.migration import migration, MigrationScriptBuilder +from pydevlake.model import Connection as BaseConnection, Model, ToolModel, ToolScope, ToolTable + + +@migration(20240604000001, name="initialize schemas for GLPI") +def init_schemas(b: MigrationScriptBuilder): + class Connection(BaseConnection): + endpoint: AnyUrl + token: SecretStr + secret_key: SecretStr + timezone: str + + class Entity(ToolScope): + completename: Optional[str] + + class ScopeConfig(ToolTable, Model): + name: str = Field(default="default") + entities: Optional[str] + connection_id: Optional[int] + collect_assignees: bool = True + collect_followups: bool = False + collect_ticket_links: bool = False + collect_logs: bool = False + collect_tasks: bool = False + collect_solutions: bool = False + tickets_created_after: Optional[datetime.datetime] = None + + class Ticket(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + content: Optional[str] + date: Optional[datetime.datetime] + closedate: Optional[datetime.datetime] + solvedate: Optional[datetime.datetime] + date_mod: Optional[datetime.datetime] + date_creation: Optional[datetime.datetime] + status: Optional[int] + type: Optional[int] + urgency: Optional[int] + impact: Optional[int] + priority: Optional[int] + entities_id: Optional[str] + itilcategories_id: Optional[int] + category_name: Optional[str] + locations_id: Optional[int] + location_name: Optional[str] + suppliers_id: Optional[int] + supplier_name: Optional[str] + requesttypes_id: Optional[int] + requesttype_name: Optional[str] + users_id_recipient: Optional[int] + users_id_lastupdater: Optional[int] + time_to_resolve: Optional[datetime.datetime] + time_to_own: Optional[datetime.datetime] + internal_time_to_resolve: Optional[datetime.datetime] + internal_time_to_own: Optional[datetime.datetime] + actiontime: Optional[int] + solve_delay_stat: Optional[int] + takeintoaccount_delay_stat: Optional[int] + requester_name: Optional[str] + assignee_id: Optional[int] + assignee_name: Optional[str] + parent_ticket_id: Optional[int] + solution_content: Optional[str] + plugin_fields_json: Optional[str] + + class TicketUser(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + users_id: Optional[int] + user_name: Optional[str] + type: Optional[int] + + class TicketGroup(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + groups_id: Optional[int] + group_name: Optional[str] + type: Optional[int] + + class Followup(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + items_id: Optional[int] + users_id: Optional[int] + content: Optional[str] + date: Optional[datetime.datetime] + date_mod: Optional[datetime.datetime] + is_private: Optional[int] + + class TicketTask(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + users_id: Optional[int] + content: Optional[str] + begin: Optional[datetime.datetime] + end: Optional[datetime.datetime] + actiontime: Optional[int] + + class TicketLog(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + items_id: Optional[int] + itemtype: Optional[str] + itemtype_link: Optional[str] + linked_action: Optional[int] + user_name: Optional[str] + date_mod: Optional[datetime.datetime] + id_search_option: Optional[int] + old_value: Optional[str] + new_value: Optional[str] + + class User(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + realname: Optional[str] + firstname: Optional[str] + email: Optional[str] + + class Group(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + completename: Optional[str] + + class ITILCategory(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + completename: Optional[str] + + class Location(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + completename: Optional[str] + + class Supplier(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + + class RequestType(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + + class TicketLabelRecord(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + label_name: Optional[str] + + class TicketCustomField(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + field_id: Optional[str] + field_value: Optional[str] + + class TicketLink(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + tickets_id_1: Optional[int] + tickets_id_2: Optional[int] + link: Optional[int] + + class ITILSolution(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + content: Optional[str] + + b.create_tables( + Connection, + Entity, + ScopeConfig, + Ticket, + TicketUser, + TicketGroup, + Followup, + TicketTask, + TicketLog, + User, + Group, + ITILCategory, + Location, + Supplier, + RequestType, + TicketLabelRecord, + TicketCustomField, + TicketLink, + ITILSolution, + ) diff --git a/backend/python/plugins/glpi/glpi/models.py b/backend/python/plugins/glpi/glpi/models.py new file mode 100644 index 00000000000..8b949090282 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/models.py @@ -0,0 +1,242 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from typing import Optional + +from pydantic import AnyUrl, SecretStr, validator +from sqlalchemy import BigInteger, Column + +from pydevlake import Field, ScopeConfig as BaseScopeConfig +from pydevlake.model import Connection as BaseConnection, ToolModel, ToolScope + +from glpi.migrations import * +from glpi.timezone import validate_timezone_name + + +class Connection(BaseConnection): + endpoint: AnyUrl + token: SecretStr + secret_key: SecretStr + timezone: str + + @validator('timezone') + def validate_timezone(cls, timezone: str) -> str: + return validate_timezone_name(timezone) + + +class ScopeConfig(BaseScopeConfig): + collect_assignees: bool = True + collect_followups: bool = False + collect_ticket_links: bool = False + collect_logs: bool = False + collect_tasks: bool = False + collect_solutions: bool = False + + +class Entity(ToolScope, table=True): + completename: Optional[str] + + +GLPI_TICKET_STATUS = { + 1: "New", + 2: "Processing (assigned)", + 3: "Processing (planned)", + 4: "Pending", + 5: "Solved", + 6: "Closed", +} + +GLPI_TICKET_STATUS_TO_DEVLAKE = { + 1: 'TODO', + 2: 'IN_PROGRESS', + 3: 'IN_PROGRESS', + 4: 'IN_PROGRESS', + 5: 'DONE', + 6: 'DONE', +} + +GLPI_TICKET_TYPE = { + 1: "Incident", + 2: "Request", +} + +GLPI_PRIORITY = { + 1: "Very Low", + 2: "Low", + 3: "Medium", + 4: "High", + 5: "Very High", + 6: "Major", +} + +GLPI_TICKET_LINK_TYPES = { + 1: 'linked', + 2: 'duplicate', + 3: 'parent', + 4: 'child', +} + + +class User(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + realname: Optional[str] + firstname: Optional[str] + email: Optional[str] + + +class Ticket(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + content: Optional[str] + date: Optional[datetime.datetime] + closedate: Optional[datetime.datetime] + solvedate: Optional[datetime.datetime] + date_mod: Optional[datetime.datetime] + date_creation: Optional[datetime.datetime] + status: Optional[int] + type: Optional[int] + urgency: Optional[int] + impact: Optional[int] + priority: Optional[int] + entities_id: Optional[str] + itilcategories_id: Optional[int] + category_name: Optional[str] + locations_id: Optional[int] + location_name: Optional[str] + suppliers_id: Optional[int] + supplier_name: Optional[str] + requesttypes_id: Optional[int] + requesttype_name: Optional[str] + users_id_recipient: Optional[int] + users_id_lastupdater: Optional[int] + time_to_resolve: Optional[datetime.datetime] + time_to_own: Optional[datetime.datetime] + internal_time_to_resolve: Optional[datetime.datetime] + internal_time_to_own: Optional[datetime.datetime] + actiontime: Optional[int] + solve_delay_stat: Optional[int] + takeintoaccount_delay_stat: Optional[int] + requester_name: Optional[str] + assignee_id: Optional[int] + assignee_name: Optional[str] + parent_ticket_id: Optional[int] + solution_content: Optional[str] + plugin_fields_json: Optional[str] + + +class ITILCategory(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + completename: Optional[str] + + +class Location(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + completename: Optional[str] + + +class Supplier(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + + +class RequestType(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + + +class TicketLabelRecord(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + label_name: Optional[str] + + +class TicketCustomField(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + field_id: Optional[str] + field_value: Optional[str] + + +class TicketUser(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + users_id: Optional[int] + user_name: Optional[str] + type: Optional[int] + + +class TicketGroup(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + groups_id: Optional[int] + group_name: Optional[str] + type: Optional[int] + + +class Followup(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + items_id: Optional[int] + users_id: Optional[int] + content: Optional[str] + date: Optional[datetime.datetime] + date_mod: Optional[datetime.datetime] + is_private: Optional[int] + + +class TicketTask(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + users_id: Optional[int] + content: Optional[str] + begin: Optional[datetime.datetime] + end: Optional[datetime.datetime] + actiontime: Optional[int] + + +class TicketLog(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + items_id: Optional[int] + itemtype: Optional[str] + itemtype_link: Optional[str] + linked_action: Optional[int] + user_name: Optional[str] + date_mod: Optional[datetime.datetime] + id_search_option: Optional[int] + old_value: Optional[str] + new_value: Optional[str] + + +class Group(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + name: Optional[str] + completename: Optional[str] + + +class TicketLink(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + tickets_id_1: Optional[int] = Field(default=None, sa_column=Column('tickets_id1', BigInteger)) + tickets_id_2: Optional[int] = Field(default=None, sa_column=Column('tickets_id2', BigInteger)) + link: Optional[int] + + +class ITILSolution(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + tickets_id: Optional[int] + content: Optional[str] diff --git a/backend/python/plugins/glpi/glpi/parsing.py b/backend/python/plugins/glpi/glpi/parsing.py new file mode 100644 index 00000000000..417eb65d6cc --- /dev/null +++ b/backend/python/plugins/glpi/glpi/parsing.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Optional, Union + +from glpi.models import User +from pydevlake.model import domain_id + +# GLPI may return dropdown/user fields as bare ids or as "Display Name (123)" +# when expand_dropdowns=true; split that combined label into name and numeric id. +_USER_SUFFIX_RE = re.compile(r'^(.*?) \((\d+)\)$') + + +def on_demand_id_from_url(raw: dict, resource: str) -> Optional[int]: + item_id = raw.get('id') + if item_id is not None: + return int(item_id) + match = re.search(rf'/{resource}/(\d+)(?:/|$)', raw.get('x_request_url', '')) + return int(match.group(1)) if match else None + + +def parse_glpi_user(value: Optional[str]) -> tuple[Optional[str], Optional[str]]: + if not value: + return None, None + value = str(value).strip() + if not value or value == '\xa0': + return None, None + match = _USER_SUFFIX_RE.match(value) + return (match.group(1).strip(), match.group(2)) if match else (value, None) + + +def parse_glpi_dropdown(value: Optional[Union[str, int]]) -> tuple[Optional[str], Optional[int]]: + if value is None: + return None, None + if isinstance(value, int): + return None, value + text = str(value).strip() + if not text or text == '\xa0': + return None, None + name, id_str = parse_glpi_user(text) + if id_str: + return name, int(id_str) + if text.isdigit(): + return None, int(text) + return text, None + + +def user_account_id(connection_id: int, user_id: Optional[Union[int, str]]) -> Optional[str]: + return domain_id(User, connection_id, int(user_id)) if user_id is not None else None diff --git a/backend/python/plugins/glpi/glpi/streams/__init__.py b/backend/python/plugins/glpi/glpi/streams/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/streams/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/plugins/glpi/glpi/streams/groups.py b/backend/python/plugins/glpi/glpi/streams/groups.py new file mode 100644 index 00000000000..88f2fce335e --- /dev/null +++ b/backend/python/plugins/glpi/glpi/streams/groups.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable + +from glpi.api import GlpiAPI +from glpi.collection import prepare_on_demand_raw, referenced_group_ids +from glpi.models import Group +from glpi.parsing import on_demand_id_from_url +from glpi.timezone import utc_now +import pydevlake.domain_layer.crossdomain as crossdomain +from pydevlake import Stream, DomainType +from pydevlake.context import Context +from pydevlake.extractor import autoextract +from pydevlake.subtasks import Collector +from sqlmodel import Session + + +class OnDemandGroupsCollector(Collector): + + def fetch(self, state, session: Session, ctx: Context): + from glpi.main import GlpiPlugin + + group_ids = referenced_group_ids(session, GlpiPlugin(), ctx) + if not group_ids: + return + with GlpiAPI(ctx.connection) as api: + for group_id in sorted(group_ids): + if group_id <= 0: + continue + prepared = prepare_on_demand_raw( + api.get_item('Group', group_id), group_id, 'Group', api.base_url, + ) + if prepared is None: + continue + yield prepared, state + + +class Groups(Stream): + tool_model = Group + domain_types = [DomainType.CROSS, DomainType.TICKET] + domain_models = [crossdomain.Account] + + def __init__(self, plugin_name: str): + super().__init__(plugin_name) + self.collector = OnDemandGroupsCollector(self) + + def collect(self, state, context) -> Iterable[tuple[object, dict]]: + yield from () + + def extract(self, raw: dict) -> Group: + raw = dict(raw) + item_id = on_demand_id_from_url(raw, 'Group') + if item_id is not None: + raw.setdefault('id', item_id) + return autoextract(raw, self.tool_model) + + def convert(self, group: Group, ctx) -> Iterable[object]: + display_name = group.completename or group.name or str(group.id) + yield crossdomain.Account( + email=f'group-{group.id}@glpi.local', + full_name=display_name, + user_name=group.name or str(group.id), + avatar_url='', + organization='', + created_date=utc_now(), + status=1, + ) diff --git a/backend/python/plugins/glpi/glpi/streams/ticket_domain.py b/backend/python/plugins/glpi/glpi/streams/ticket_domain.py new file mode 100644 index 00000000000..3767d701c2f --- /dev/null +++ b/backend/python/plugins/glpi/glpi/streams/ticket_domain.py @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Generator, Iterable, Optional + +from glpi.models import ( + ITILSolution, + Ticket, + TicketLink, + TicketUser, + GLPI_TICKET_STATUS, + GLPI_TICKET_STATUS_TO_DEVLAKE, + GLPI_TICKET_TYPE, + GLPI_PRIORITY, +) +from glpi.parsing import user_account_id +from glpi.timezone import connection_timezone, to_utc +from sqlalchemy import or_ +import pydevlake.domain_layer.ticket as ticket +from pydevlake import Stream, DomainType, domain_id +from pydevlake.context import Context +from pydevlake.model import ToolModel +from pydevlake.subtasks import Convertor +from sqlmodel import Session + + +class TicketDomainConvertor(Convertor): + + def process(self, tool_model: ToolModel, session: Session, ctx: Context): + self._enrich_ticket_from_tool_records(session, ctx, tool_model) + res = self.stream.convert(tool_model, ctx) + if isinstance(res, Generator): + for each in res: + self._save(tool_model, each, session, ctx.connection.id) + else: + self._save(tool_model, res, session, ctx.connection.id) + + def _enrich_ticket_from_tool_records(self, session: Session, ctx: Context, ticket_model: Ticket) -> None: + assigned_user_type = 2 + assignees = session.query(TicketUser).where( + TicketUser.connection_id == ctx.connection.id, + TicketUser.tickets_id == ticket_model.id, + TicketUser.type == assigned_user_type, + ).all() + if assignees: + row = assignees[0] + ticket_model.assignee_id = row.users_id + ticket_model.assignee_name = row.user_name + + links = session.query(TicketLink).where( + TicketLink.connection_id == ctx.connection.id, + or_( + TicketLink.tickets_id_1 == ticket_model.id, + TicketLink.tickets_id_2 == ticket_model.id, + ), + ).all() + link_dicts = [ + {'link': link.link, 'tickets_id_1': link.tickets_id_1, 'tickets_id_2': link.tickets_id_2} + for link in links + ] + parent_id = self._parent_ticket_id_from_links(link_dicts, ticket_model.id) + if parent_id is not None: + ticket_model.parent_ticket_id = parent_id + + solutions = session.query(ITILSolution).where( + ITILSolution.connection_id == ctx.connection.id, + ITILSolution.tickets_id == ticket_model.id, + ).order_by(ITILSolution.id).all() + if solutions: + ticket_model.solution_content = solutions[-1].content + + def _parent_ticket_id_from_links(self, links: list, ticket_id: int) -> Optional[int]: + glpi_link_son_of = 3 + glpi_link_parent_of = 4 + for link in links: + link_type = link.get('link') + t1, t2 = link.get('tickets_id_1'), link.get('tickets_id_2') + if link_type == glpi_link_son_of and t1 == ticket_id: + return t2 + if link_type == glpi_link_parent_of and t2 == ticket_id: + return t1 + return None + + +class TicketDomain(Stream): + tool_model = Ticket + domain_types = [DomainType.TICKET] + domain_models = [ + ticket.Issue, + ticket.BoardIssue, + ticket.IssueLabel, + ] + + def __init__(self, plugin_name: str): + super().__init__(plugin_name) + self.convertor = TicketDomainConvertor(self) + + @property + def subtasks(self): + return [self.convertor] + + def convert(self, t: Ticket, ctx: Context) -> Iterable[object]: + board_id = ctx.scope.domain_id() + tz = connection_timezone(ctx.connection.timezone) + glpi_type_to_devlake = { + 1: 'INCIDENT', + 2: 'TASK', + } + issue_id = domain_id(Ticket, ctx.connection.id, t.id) + issue = ticket.Issue( + url=self._ticket_url(ctx.connection.endpoint, t.id), + issue_key=str(t.id), + title=t.name or '', + description=self._issue_description(t.content, t.solution_content), + type=glpi_type_to_devlake.get(t.type, 'TASK'), + original_type=GLPI_TICKET_TYPE.get(t.type) or (str(t.type) if t.type else None), + status=GLPI_TICKET_STATUS_TO_DEVLAKE.get(t.status, 'OTHER'), + original_status=GLPI_TICKET_STATUS.get(t.status), + created_date=to_utc(t.date or t.date_creation, tz), + updated_date=to_utc(t.date_mod, tz), + resolution_date=to_utc(t.solvedate or t.closedate, tz), + due_date=to_utc(t.time_to_resolve, tz), + creator_id=user_account_id(ctx.connection.id, t.users_id_recipient), + creator_name=t.requester_name, + assignee_id=user_account_id(ctx.connection.id, t.assignee_id), + assignee_name=t.assignee_name, + parent_issue_id=( + domain_id(Ticket, ctx.connection.id, t.parent_ticket_id) + if t.parent_ticket_id else None + ), + is_subtask=t.parent_ticket_id is not None, + priority=GLPI_PRIORITY.get(t.priority), + urgency=GLPI_PRIORITY.get(t.urgency), + severity=GLPI_PRIORITY.get(t.impact), + component=t.category_name, + original_project=t.entities_id, + lead_time_minutes=t.solve_delay_stat // 60 if t.solve_delay_stat else None, + ) + yield issue + yield ticket.BoardIssue(board_id=board_id, issue_id=issue.id) + if t.category_name: + yield ticket.IssueLabel(issue_id=issue_id, label_name=t.category_name) + + def _issue_description(self, content: Optional[str], solution: Optional[str]) -> Optional[str]: + if not solution: + return content + if not content: + return solution + return f'{content}\n\n--- Solution ---\n{solution}' + + @staticmethod + def _ticket_url(endpoint: str, ticket_id: int) -> str: + base = str(endpoint).rstrip('/') + return f'{base}/front/ticket.form.php?id={ticket_id}' diff --git a/backend/python/plugins/glpi/glpi/streams/ticket_substreams.py b/backend/python/plugins/glpi/glpi/streams/ticket_substreams.py new file mode 100644 index 00000000000..caeef8bed94 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/streams/ticket_substreams.py @@ -0,0 +1,471 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, Optional, Union + +from glpi.api import GlpiAPI +from glpi.models import ( + Followup, + GLPI_TICKET_LINK_TYPES, + GLPI_TICKET_STATUS, + GLPI_TICKET_STATUS_TO_DEVLAKE, + GLPI_PRIORITY, + GLPI_TICKET_TYPE, + Group, + ITILSolution, + Ticket, + TicketGroup, + TicketLink, + TicketLog, + TicketTask, + TicketUser, +) +from glpi.parsing import parse_glpi_dropdown, parse_glpi_user, user_account_id +from glpi.streams.tickets import Tickets +from glpi.timezone import connection_timezone, to_utc +from pydevlake import DomainType, Substream, domain_id +from pydevlake.context import Context +from pydevlake.extractor import autoextract +from pydevlake.subtasks import Collector, SubstreamCollector +from sqlmodel import Session +import pydevlake.domain_layer.ticket as ticket + + +# GLPI REST bug: GET /Ticket/{id}/Ticket_Ticket ignores the parent ticket id and +# returns every link in the instance (other sub-item endpoints filter correctly). +# Workaround: paginate Ticket_Ticket once, then filter to collected ticket ids. +# https://forum.glpi-project.org/viewtopic.php?id=288309 +class TicketLinksCollector(Collector): + + def fetch(self, state, session: Session, ctx: Context): + if not ctx.scope_config.collect_ticket_links: + return + ticket_ids = self._collected_ticket_ids(session, self._params(ctx)) + if not ticket_ids: + return + with GlpiAPI(ctx.connection) as api: + for raw in api.iter_ticket_links(): + if not self._link_involves_tickets(raw, ticket_ids): + continue + raw['x_request_url'] = f'{api.base_url}/apirest.php/Ticket_Ticket' + raw['x_request_input'] = {} + yield raw, state + + @staticmethod + def _collected_ticket_ids(session: Session, params: str) -> set[int]: + return { + ticket.id + for ticket in session.query(Ticket).where( + Ticket.raw_data_params == params, + ).all() + } + + @staticmethod + def _link_involves_tickets(raw: dict, ticket_ids: set[int]) -> bool: + linked = ( + raw.get('tickets_id_1') or raw.get('tickets_id1'), + raw.get('tickets_id_2') or raw.get('tickets_id2'), + ) + return any(int(ticket_id) in ticket_ids for ticket_id in linked if ticket_id is not None) + + +class GlpiSubstreamCollector(SubstreamCollector): + + def fetch(self, state, session, ctx: Context): + parent_model = self.stream.parent_stream.tool_model + query = session.query(parent_model).where( + parent_model.raw_data_params == self._params(ctx), + ) + parents = query.all() + if not parents: + return + with GlpiAPI(ctx.connection) as api: + for parent in parents: + yield from self.stream.collect(state, ctx, parent, api) + + +class GlpiTicketSubstream(Substream): + parent_stream = Tickets + glpi_resource: str + optional_collect_flag: Optional[str] = None + optional_collect_default: bool = False + + def __init__(self, plugin_name: str): + super().__init__(plugin_name) + self.collector = GlpiSubstreamCollector(self) + + def collect( + self, + state, + context, + parent: Ticket, + api: GlpiAPI, + ) -> Iterable[tuple[object, dict]]: + if self.optional_collect_flag is not None and not getattr( + context.scope_config, + self.optional_collect_flag, + self.optional_collect_default, + ): + return + ticket_id = parent.id + for raw in api.ticket_subitems(ticket_id, self.glpi_resource): + raw['tickets_id'] = ticket_id + if self.glpi_resource == 'ITILFollowup': + raw['items_id'] = ticket_id + raw['x_request_url'] = f'{api.base_url}/apirest.php/Ticket/{ticket_id}/{self.glpi_resource}' + raw['x_request_input'] = {'TicketId': ticket_id} + yield raw, state + + +class TicketUsers(GlpiTicketSubstream): + tool_model = TicketUser + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueAssignee] + glpi_resource = 'Ticket_User' + optional_collect_flag = 'collect_assignees' + optional_collect_default = True + + def extract(self, raw: dict) -> TicketUser: + model = autoextract(raw, self.tool_model) + name, user_id = parse_glpi_dropdown(raw.get('users_id')) + model.users_id = user_id + model.user_name = name + return model + + def convert(self, tu: TicketUser, ctx: Context) -> Iterable[ticket.IssueAssignee]: + result = self._convert_ticket_user_assignee(tu, ctx) + if result is not None: + yield result + + def _convert_ticket_user_assignee(self, tu: TicketUser, ctx: Context) -> Optional[ticket.IssueAssignee]: + assigned_user_type = 2 + if tu.type != assigned_user_type or tu.tickets_id is None: + return None + user_id = tu.users_id + if user_id is None: + return None + issue_id = domain_id(Ticket, ctx.connection.id, tu.tickets_id) + return ticket.IssueAssignee( + issue_id=issue_id, + assignee_id=user_account_id(ctx.connection.id, user_id), + assignee_name=tu.user_name, + ) + + +def _group_account_id(connection_id: int, group_id: Optional[Union[int, str]]) -> Optional[str]: + return domain_id(Group, connection_id, int(group_id)) if group_id is not None else None + + +class TicketGroups(GlpiTicketSubstream): + tool_model = TicketGroup + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueAssignee] + glpi_resource = 'Group_Ticket' + optional_collect_flag = 'collect_assignees' + optional_collect_default = True + + def extract(self, raw: dict) -> TicketGroup: + model = autoextract(raw, self.tool_model) + name, group_id = parse_glpi_dropdown(raw.get('groups_id')) + model.groups_id = group_id + model.group_name = name + return model + + def convert(self, tg: TicketGroup, ctx: Context) -> Iterable[ticket.IssueAssignee]: + result = self._convert_ticket_group_assignee(tg, ctx) + if result is not None: + yield result + + def _convert_ticket_group_assignee(self, tg: TicketGroup, ctx: Context) -> Optional[ticket.IssueAssignee]: + assigned_group_type = 2 + if tg.type != assigned_group_type or tg.tickets_id is None or tg.groups_id is None: + return None + issue_id = domain_id(Ticket, ctx.connection.id, tg.tickets_id) + return ticket.IssueAssignee( + issue_id=issue_id, + assignee_id=_group_account_id(ctx.connection.id, tg.groups_id), + assignee_name=tg.group_name, + ) + + +_GLPI_TICKET_STATUS_NAME_TO_DEVLAKE = { + GLPI_TICKET_STATUS[code]: status + for code, status in GLPI_TICKET_STATUS_TO_DEVLAKE.items() +} + + +class TicketFollowups(GlpiTicketSubstream): + tool_model = Followup + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueComment] + glpi_resource = 'ITILFollowup' + optional_collect_flag = 'collect_followups' + + def convert(self, followup: Followup, ctx: Context) -> Iterable[ticket.IssueComment]: + result = self._convert_followup_comment(followup, ctx) + if result is not None: + yield result + + def _convert_followup_comment(self, followup: Followup, ctx: Context) -> Optional[ticket.IssueComment]: + if followup.is_private in (1, '1', True) or followup.items_id is None: + return None + tz = connection_timezone(ctx.connection.timezone) + issue_id = domain_id(Ticket, ctx.connection.id, followup.items_id) + comment = ticket.IssueComment( + issue_id=issue_id, + body=followup.content, + account_id=user_account_id(ctx.connection.id, followup.users_id), + created_date=to_utc(followup.date, tz), + updated_date=to_utc(followup.date_mod, tz), + ) + if followup.id is not None: + comment.id = domain_id(Followup, ctx.connection.id, followup.id) + return comment + + +class TicketTasks(GlpiTicketSubstream): + tool_model = TicketTask + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueWorklog] + glpi_resource = 'TicketTask' + optional_collect_flag = 'collect_tasks' + + def convert(self, task: TicketTask, ctx: Context) -> Iterable[ticket.IssueWorklog]: + result = self._convert_ticket_task_worklog(task, ctx) + if result is not None: + yield result + + def _convert_ticket_task_worklog(self, task: TicketTask, ctx: Context) -> Optional[ticket.IssueWorklog]: + if not task.actiontime or task.tickets_id is None: + return None + tz = connection_timezone(ctx.connection.timezone) + issue_id = domain_id(Ticket, ctx.connection.id, task.tickets_id) + worklog = ticket.IssueWorklog( + issue_id=issue_id, + author_id=user_account_id(ctx.connection.id, task.users_id), + comment=task.content, + time_spent_minutes=int(task.actiontime) // 60, + started_date=to_utc(task.begin, tz), + logged_date=to_utc(task.begin, tz), + ) + if task.id is not None: + worklog.id = domain_id(TicketTask, ctx.connection.id, task.id) + return worklog + + +class TicketLogs(GlpiTicketSubstream): + tool_model = TicketLog + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueChangelogs] + glpi_resource = 'Log' + optional_collect_flag = 'collect_logs' + + def convert(self, log: TicketLog, ctx: Context) -> Iterable[ticket.IssueChangelogs]: + result = self._convert_ticket_log_changelog(log, ctx) + if result is not None: + yield result + + def _convert_ticket_log_changelog(self, log: TicketLog, ctx: Context) -> Optional[ticket.IssueChangelogs]: + if log.tickets_id is None: + return None + resolved = self._devlake_changelog_from_glpi_ticket_log(log) + if not resolved: + return None + tz = connection_timezone(ctx.connection.timezone) + issue_id = domain_id(Ticket, ctx.connection.id, log.tickets_id) + author_name, author_id = parse_glpi_user(log.user_name) + itemtype_link = resolved.get('itemtype_link') + from_value = resolved['from_value'] + to_value = resolved['to_value'] + if resolved['field_name'] in ('assignee', 'requester', 'observer'): + from_value = self._account_id_from_glpi_ref( + ctx.connection.id, resolved['original_from_value'], itemtype_link + ) + to_value = self._account_id_from_glpi_ref( + ctx.connection.id, resolved['original_to_value'], itemtype_link + ) + changelog = ticket.IssueChangelogs( + issue_id=issue_id, + author_id=user_account_id(ctx.connection.id, author_id), + author_name=author_name, + field_id=resolved['field_id'], + field_name=resolved['field_name'], + original_from_value=resolved['original_from_value'], + original_to_value=resolved['original_to_value'], + from_value=from_value, + to_value=to_value, + created_date=to_utc(log.date_mod, tz), + ) + if log.id is not None: + changelog.id = domain_id(TicketLog, ctx.connection.id, log.id) + return changelog + + def _devlake_changelog_from_glpi_ticket_log(self, log: TicketLog) -> Optional[dict]: + field_name = self._changelog_field_name(log) + if not field_name: + return None + + old_value = self._clean_changelog_value(log.old_value) + new_value = self._clean_changelog_value(log.new_value) + if not old_value and not new_value: + return None + + result = { + 'field_name': field_name, + 'field_id': str(log.id_search_option) if log.id_search_option is not None else None, + 'original_from_value': old_value, + 'original_to_value': new_value, + 'from_value': old_value, + 'to_value': new_value, + } + + if field_name == 'status': + result['from_value'] = self._devlake_status_from_glpi_name(old_value) + result['to_value'] = self._devlake_status_from_glpi_name(new_value) + elif field_name in ('priority', 'urgency', 'impact'): + result['from_value'] = self._devlake_priority_from_glpi_name(old_value) + result['to_value'] = self._devlake_priority_from_glpi_name(new_value) + elif field_name == 'type': + result['from_value'] = self._devlake_type_from_glpi_name(old_value) + result['to_value'] = self._devlake_type_from_glpi_name(new_value) + elif field_name in ('assignee', 'requester', 'observer'): + _, from_id = parse_glpi_user(old_value) + _, to_id = parse_glpi_user(new_value) + result['from_value'] = from_id + result['to_value'] = to_id + result['original_from_value'] = from_id or old_value + result['original_to_value'] = to_id or new_value + result['itemtype_link'] = log.itemtype_link or 'User' + + return result + + def _changelog_field_name(self, log: TicketLog) -> Optional[str]: + history_add_relation = 15 + history_del_relation = 16 + if log.linked_action in (history_add_relation, history_del_relation): + if log.itemtype_link == 'Group': + return 'assignee' + return { + 4: 'requester', + 5: 'assignee', + 66: 'observer', + }.get(log.id_search_option) + if log.linked_action and log.linked_action != 0: + return None + return { + 3: 'priority', + 4: 'requester', + 5: 'assignee', + 7: 'category', + 10: 'urgency', + 11: 'impact', + 12: 'status', + 14: 'type', + 66: 'observer', + }.get(log.id_search_option) + + def _clean_changelog_value(self, value: Optional[str]) -> Optional[str]: + if value is None: + return None + value = value.strip() + if not value or value == '\xa0': + return None + return value + + def _devlake_status_from_glpi_name(self, value: Optional[str]) -> Optional[str]: + if not value: + return None + text = value.strip() + if text.isdigit(): + return GLPI_TICKET_STATUS_TO_DEVLAKE.get(int(text)) + return _GLPI_TICKET_STATUS_NAME_TO_DEVLAKE.get(text) + + def _devlake_priority_from_glpi_name(self, value: Optional[str]) -> Optional[str]: + if not value: + return None + text = value.strip() + if text.isdigit(): + return GLPI_PRIORITY.get(int(text)) + return text if text in GLPI_PRIORITY.values() else text + + def _devlake_type_from_glpi_name(self, value: Optional[str]) -> Optional[str]: + if not value: + return None + text = value.strip() + glpi_type_name_to_devlake = { + "Incident": 'INCIDENT', + "Request": 'TASK', + } + if text.isdigit(): + name = GLPI_TICKET_TYPE.get(int(text)) + return glpi_type_name_to_devlake.get(name) if name else None + return glpi_type_name_to_devlake.get(text) + + + def _account_id_from_glpi_ref( + self, + connection_id: int, + value: Optional[str], + itemtype_link: Optional[str] = None, + ) -> Optional[str]: + if not value: + return None + if itemtype_link == 'Group': + _, group_id = parse_glpi_dropdown(value) + return _group_account_id(connection_id, group_id) + _, user_id = parse_glpi_user(value) + if user_id is None and str(value).strip().isdigit(): + user_id = str(value).strip() + return user_account_id(connection_id, user_id) + + +class TicketLinks(Substream): + # Uses TicketLinksCollector; see comment there for why this is not a per-ticket substream. + parent_stream = Tickets + tool_model = TicketLink + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueRelationship] + + def __init__(self, plugin_name: str): + super().__init__(plugin_name) + self.collector = TicketLinksCollector(self) + + def convert(self, link: TicketLink, ctx: Context) -> Iterable[ticket.IssueRelationship]: + result = self._convert_ticket_link(link, ctx) + if result is not None: + yield result + + def _convert_ticket_link(self, link: TicketLink, ctx: Context) -> Optional[ticket.IssueRelationship]: + t1, t2 = link.tickets_id_1, link.tickets_id_2 + if not (t1 and t2): + return None + link_type = GLPI_TICKET_LINK_TYPES.get(link.link, str(link.link or 'linked')) + return ticket.IssueRelationship( + source_issue_id=domain_id(Ticket, ctx.connection.id, t1), + target_issue_id=domain_id(Ticket, ctx.connection.id, t2), + original_type=link_type, + ) + + +class TicketSolutions(GlpiTicketSubstream): + tool_model = ITILSolution + domain_types = [DomainType.TICKET] + domain_models = [] + glpi_resource = 'ITILSolution' + optional_collect_flag = 'collect_solutions' + + @property + def subtasks(self): + return [self.collector, self.extractor] diff --git a/backend/python/plugins/glpi/glpi/streams/tickets.py b/backend/python/plugins/glpi/glpi/streams/tickets.py new file mode 100644 index 00000000000..d830ca610fd --- /dev/null +++ b/backend/python/plugins/glpi/glpi/streams/tickets.py @@ -0,0 +1,282 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from dataclasses import dataclass, field +from typing import Iterable, Optional, Union + +from glpi.api import GlpiAPI +from glpi.models import Entity, Ticket, User +from glpi.parsing import parse_glpi_dropdown +from glpi.timezone import connection_timezone, glpi_search_time_from_iso +from pydevlake import Stream, DomainType +from pydevlake.collection_time import collection_time_after +from pydevlake.context import Context +from pydevlake.extractor import autoextract +from pydevlake.subtasks import Extractor +from sqlmodel import Session + +_LOOKUP_KEY = '_glpi_lookup' + + +@dataclass +class GlpiLookup: + users_by_name: dict[str, int] = field(default_factory=dict) + + @classmethod + def build(cls, session: Session, connection_id: int) -> 'GlpiLookup': + users_by_name: dict[str, int] = {} + for row in session.query(User).where(User.connection_id == connection_id): + if row.id is None or not row.name: + continue + users_by_name[str(row.name).strip().lower()] = int(row.id) + return cls(users_by_name=users_by_name) + + +def _set_lookup(context: Context, lookup: GlpiLookup) -> None: + context.options[_LOOKUP_KEY] = lookup + + +def _lookup(context: Context) -> GlpiLookup: + stored = context.options.get(_LOOKUP_KEY) + if stored is None: + stored = GlpiLookup() + _set_lookup(context, stored) + return stored + + +class TicketExtractor(Extractor): + + def fetch(self, state, session: Session, ctx: Context): + _set_lookup(ctx, GlpiLookup.build(session, ctx.connection.id)) + raw_model = self.stream.raw_model(session) + params = self._params(ctx) + last_id = 0 + batch_size = 200 + while True: + batch = ( + session.query(raw_model) + .where(raw_model.params == params) + .where(raw_model.id > last_id) + .order_by(raw_model.id) + .limit(batch_size) + .all() + ) + if not batch: + break + for raw in batch: + last_id = raw.id + yield raw, state + + def process(self, raw, session: Session, ctx: Context): + tool_model = self.stream.extract(json.loads(raw.data), ctx) + tool_model.set_raw_origin(raw) + tool_model.connection_id = ctx.connection.id + tool_model.set_updated_at() + session.merge(tool_model) + + +def _glpi_native_ticket_keys() -> set[str]: + ticket_enrichment_fields = { + 'category_name', 'location_name', 'supplier_name', 'requesttype_name', + 'requester_name', 'assignee_id', 'assignee_name', + 'parent_ticket_id', 'solution_content', 'plugin_fields_json', + } + glpi_ticket_ignored_keys = {'links'} + return { + name for name, field_info in Ticket.__fields__.items() + if name not in ticket_enrichment_fields + and not field_info.field_info.extra.get('source') + } | glpi_ticket_ignored_keys + + +_GLPI_NATIVE_TICKET_KEYS = _glpi_native_ticket_keys() + + +class Tickets(Stream): + tool_model = Ticket + domain_types = [DomainType.TICKET] + domain_models = [] + + def __init__(self, plugin_name: str): + super().__init__(plugin_name) + self.extractor = TicketExtractor(self) + + @property + def subtasks(self): + return [self.collector, self.extractor] + + def collect(self, state, context) -> Iterable[tuple[object, dict]]: + entity: Entity = context.scope + run_state = dict(state) + use_incremental = context.options.get('incremental') + updated_since = run_state.get('updated_since') if use_incremental else None + created_after = None + if not use_incremental: + time_after = collection_time_after(context) + if time_after: + created_after = glpi_search_time_from_iso( + time_after.isoformat(), + connection_timezone(context.connection.timezone), + ) + catalog_cache: dict[str, str] = {} + user_cache: dict[int, str] = {} + with GlpiAPI(context.connection) as api: + for raw_ticket in api.iter_tickets( + entity_id=entity.id, + updated_since=updated_since, + created_after=created_after, + ): + self._track_updated_since(run_state, raw_ticket.get('date_mod')) + self._resolve_bare_catalog_fields(api, raw_ticket, catalog_cache) + self._resolve_bare_user_fields(api, raw_ticket, user_cache) + raw_ticket['x_request_url'] = f'{api.base_url}/apirest.php/Ticket' + raw_ticket['x_request_input'] = {'EntityId': entity.id} + yield raw_ticket, run_state + + @staticmethod + def _track_updated_since(state: dict, date_mod: Optional[str]) -> None: + if not date_mod: + return + current = state.get('updated_since') + if current is None or date_mod > current: + state['updated_since'] = date_mod + + @staticmethod + def _resolve_bare_catalog_fields( + api: GlpiAPI, + ticket_raw: dict, + catalog_cache: dict[str, str], + ) -> None: + catalog_fields = { + 'itilcategories_id': ('ITILCategory', ('completename', 'name')), + 'locations_id': ('Location', ('completename', 'name')), + 'suppliers_id': ('Supplier', ('name',)), + 'requesttypes_id': ('RequestType', ('name',)), + } + for field_name, (itemtype, name_fields) in catalog_fields.items(): + name, item_id = parse_glpi_dropdown(ticket_raw.get(field_name)) + if item_id is None or item_id <= 0 or name: + continue + cache_key = f'{itemtype}:{item_id}' + if cache_key not in catalog_cache: + raw = api.get_item(itemtype, item_id) + resolved = next( + (raw.get(field) for field in name_fields if raw.get(field)), + None, + ) + catalog_cache[cache_key] = resolved or str(item_id) + ticket_raw[field_name] = f'{catalog_cache[cache_key]} ({item_id})' + + @staticmethod + def _resolve_bare_user_fields( + api: GlpiAPI, + ticket_raw: dict, + user_cache: dict[int, str], + ) -> None: + for field in ('users_id_recipient', 'users_id_lastupdater'): + user_id = Tickets._bare_dropdown_id(ticket_raw.get(field)) + if user_id is None: + continue + if user_id not in user_cache: + user_cache[user_id] = Tickets._user_display_name(api.get_item('User', user_id)) + ticket_raw[field] = f'{user_cache[user_id]} ({user_id})' + + @staticmethod + def _bare_dropdown_id(value) -> Optional[int]: + name, item_id = parse_glpi_dropdown(value) + return item_id if item_id is not None and not name else None + + @staticmethod + def _user_display_name(user_raw: dict) -> str: + for field in ('realname', 'firstname', 'name'): + value = user_raw.get(field) + if value: + text = str(value).strip() + if text: + return text + return str(user_raw.get('id', '')) + + def extract(self, raw: dict, ctx: Optional[Context] = None) -> Ticket: + ticket_model = autoextract(raw, self.tool_model) + if ctx is not None: + self._apply_ticket_enrichment(ticket_model, raw, ctx) + return ticket_model + + @staticmethod + def _apply_ticket_enrichment( + ticket_model: Ticket, + raw: dict, + ctx: Context, + ) -> None: + requester_name, requester_id = Tickets._resolve_glpi_user_ref(ctx, raw.get('users_id_recipient')) + ticket_model.requester_name = requester_name + if requester_id is not None: + ticket_model.users_id_recipient = requester_id + + entity_name, entity_id = parse_glpi_dropdown(raw.get('entities_id')) + if entity_id is not None and str(entity_id) == str(ctx.scope.id): + ticket_model.entities_id = ctx.scope.completename or ctx.scope.name + elif entity_name and not entity_id: + ticket_model.entities_id = entity_name + + category_name, category_id = parse_glpi_dropdown(raw.get('itilcategories_id')) + ticket_model.itilcategories_id = category_id + ticket_model.category_name = category_name + + location_name, location_id = parse_glpi_dropdown(raw.get('locations_id')) + ticket_model.locations_id = location_id + ticket_model.location_name = location_name + + supplier_name, supplier_id = parse_glpi_dropdown(raw.get('suppliers_id')) + ticket_model.suppliers_id = supplier_id + ticket_model.supplier_name = supplier_name + + requesttype_name, requesttype_id = parse_glpi_dropdown(raw.get('requesttypes_id')) + ticket_model.requesttypes_id = requesttype_id + ticket_model.requesttype_name = requesttype_name + + plugin_fields = Tickets._extract_plugin_fields(raw) + if plugin_fields: + ticket_model.plugin_fields_json = json.dumps(plugin_fields) + + @staticmethod + def _resolve_glpi_user_ref( + context: Context, + value: Optional[Union[str, int]], + ) -> tuple[Optional[str], Optional[int]]: + name, user_id = parse_glpi_dropdown(value) + if user_id is not None: + return name, user_id + if not name: + return None, None + resolved_id = _lookup(context).users_by_name.get(name.strip().lower()) + if resolved_id is not None: + return name, resolved_id + return name, None + + @staticmethod + def _extract_plugin_fields(raw: dict) -> dict[str, str]: + return { + key: str(value) + for key, value in raw.items() + if ( + key not in _GLPI_NATIVE_TICKET_KEYS + and not key.startswith('_') + and not key.startswith('x_') + and value is not None + and value != '' + ) + } diff --git a/backend/python/plugins/glpi/glpi/streams/users.py b/backend/python/plugins/glpi/glpi/streams/users.py new file mode 100644 index 00000000000..4127524f313 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/streams/users.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable + +from glpi.api import GlpiAPI +from glpi.collection import prepare_on_demand_raw, referenced_user_ids +from glpi.models import User +from glpi.parsing import on_demand_id_from_url +from glpi.timezone import utc_now +import pydevlake.domain_layer.crossdomain as crossdomain +from pydevlake import Stream, DomainType +from pydevlake.context import Context +from pydevlake.extractor import autoextract +from pydevlake.subtasks import Collector +from sqlmodel import Session + + +class OnDemandUsersCollector(Collector): + + def fetch(self, state, session: Session, ctx: Context): + from glpi.main import GlpiPlugin + + user_ids = referenced_user_ids(session, GlpiPlugin(), ctx) + if not user_ids: + return + with GlpiAPI(ctx.connection) as api: + for user_id in sorted(user_ids): + if user_id <= 0: + continue + prepared = prepare_on_demand_raw( + api.get_item('User', user_id), user_id, 'User', api.base_url, + ) + if prepared is None: + continue + yield prepared, state + + +class Users(Stream): + tool_model = User + domain_types = [DomainType.CROSS, DomainType.TICKET] + domain_models = [crossdomain.Account] + + def __init__(self, plugin_name: str): + super().__init__(plugin_name) + self.collector = OnDemandUsersCollector(self) + + def collect(self, state, context) -> Iterable[tuple[object, dict]]: + yield from () + + def extract(self, raw: dict) -> User: + raw = dict(raw) + item_id = on_demand_id_from_url(raw, 'User') + if item_id is not None: + raw.setdefault('id', item_id) + return autoextract(raw, self.tool_model) + + def convert(self, user: User, ctx) -> Iterable[object]: + full_name = ' '.join( + part for part in [user.firstname, user.realname] if part + ).strip() or user.name or '' + yield crossdomain.Account( + email=user.email or '', + full_name=full_name, + user_name=user.name or str(user.id), + avatar_url='', + organization='', + created_date=utc_now(), + status=1, + ) diff --git a/backend/python/plugins/glpi/glpi/timezone.py b/backend/python/plugins/glpi/glpi/timezone.py new file mode 100644 index 00000000000..a8f60f86db7 --- /dev/null +++ b/backend/python/plugins/glpi/glpi/timezone.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timezone +from typing import Optional +from zoneinfo import ZoneInfo, available_timezones + + +def validate_timezone_name(timezone_name: str) -> str: + timezone_name = timezone_name.strip() + if not timezone_name: + raise ValueError('timezone is required') + if timezone_name not in available_timezones(): + raise ValueError(f'unknown timezone: {timezone_name}') + return timezone_name + + +def connection_timezone(timezone_name: str) -> ZoneInfo: + return ZoneInfo(validate_timezone_name(timezone_name)) + + +def utc_now() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) + + +def to_utc(dt: Optional[datetime], tz: ZoneInfo) -> Optional[datetime]: + if dt is None: + return None + if dt.tzinfo is not None: + return dt.astimezone(timezone.utc).replace(tzinfo=None) + return dt.replace(tzinfo=tz).astimezone(timezone.utc).replace(tzinfo=None) + + +def glpi_search_time_from_iso(iso_time: str, tz: ZoneInfo) -> str: + normalized = iso_time.replace('Z', '+00:00') + dt = datetime.fromisoformat(normalized) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(tz).strftime('%Y-%m-%d %H:%M:%S') diff --git a/backend/python/plugins/glpi/pyproject.toml b/backend/python/plugins/glpi/pyproject.toml new file mode 100644 index 00000000000..fd8cf66d609 --- /dev/null +++ b/backend/python/plugins/glpi/pyproject.toml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[tool.poetry] +name = "glpi" +version = "0.1.0" +description = "DevLake plugin for GLPI" +authors = ["Roger Abelenda <3179183+rabelenda@users.noreply.github.com>"] +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.11,<3.12" +pydevlake = { path = "../../pydevlake", develop = true } + +[tool.poetry.group.dev.dependencies] +pytest = "^9.1.1" + +[tool.pytest.ini_options] +filterwarnings = [ + "ignore:This declarative base already contains:sqlalchemy.exc.SAWarning", +] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/backend/python/plugins/glpi/run.sh b/backend/python/plugins/glpi/run.sh new file mode 100755 index 00000000000..0011f088b45 --- /dev/null +++ b/backend/python/plugins/glpi/run.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "$(dirname "$0")" +sh ../../uv.sh python . glpi/main.py "$@" diff --git a/backend/python/plugins/glpi/tests/__init__.py b/backend/python/plugins/glpi/tests/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/python/plugins/glpi/tests/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/plugins/glpi/tests/api_test.py b/backend/python/plugins/glpi/tests/api_test.py new file mode 100644 index 00000000000..9ceaceda119 --- /dev/null +++ b/backend/python/plugins/glpi/tests/api_test.py @@ -0,0 +1,191 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager +from typing import Iterator +from unittest.mock import MagicMock, Mock, patch + +from pydevlake.api import APIException + +from glpi.api import GlpiAPI, _TICKET_SORT_DATE_MOD, _TICKET_SORT_OPENING_DATE + + +def _mock_response(body, content_range='0-99/100'): + response = MagicMock() + response.json = body + response.headers = {'Content-Range': content_range} + return response + + +class PatchedGlpiAPI: + _AUTO_MOCK_METHODS = frozenset({'get', '_api_get'}) + + def __init__(self): + object.__setattr__(self, '_api', GlpiAPI(MagicMock())) + object.__setattr__(self, '_mocks', {}) + object.__setattr__(self, '_patchers', []) + + def _install_mock(self, name: str, mock: Mock) -> Mock: + self._mocks[name] = mock + patcher = patch.object(self._api, name, mock) + patcher.start() + self._patchers.append(patcher) + return mock + + def __getattr__(self, name): + if name in self._mocks: + return self._mocks[name] + if name in self._AUTO_MOCK_METHODS: + return self._install_mock(name, MagicMock()) + return getattr(self._api, name) + + def __setattr__(self, name, value): + if name.startswith('_'): + object.__setattr__(self, name, value) + elif isinstance(value, Mock): + self._install_mock(name, value) + else: + setattr(self._api, name, value) + + def __enter__(self): + return self + + def __exit__(self, *_): + for patcher in reversed(self._patchers): + patcher.stop() + + +@contextmanager +def _patched_api() -> Iterator[PatchedGlpiAPI]: + with PatchedGlpiAPI() as api: + yield api + + +def test_iter_ticket_list_caps_range_on_first_request(): + with _patched_api() as api: + api.get.return_value = _mock_response([{'id': i} for i in range(1, 6)], '0-4/5') + collected = list(api._iter_ticket_list(limit=5)) + + assert [ticket['id'] for ticket in collected] == [1, 2, 3, 4, 5] + api.get.assert_called_once() + assert api.get.call_args.kwargs['range'] == '0-4' + assert api.get.call_args.kwargs['expand_dropdowns'] == 'true' + + +def test_iter_ticket_list_stops_after_limit_across_pages(): + page_one = [{'id': i} for i in range(1, 101)] + page_two = [{'id': i} for i in range(101, 106)] + + with _patched_api() as api: + api.get.side_effect = [ + _mock_response(page_one, '0-99/105'), + _mock_response(page_two, '100-104/105'), + ] + collected = list(api._iter_ticket_list(limit=105)) + + assert len(collected) == 105 + assert api.get.call_count == 2 + assert api.get.call_args_list[0].kwargs['range'] == '0-99' + assert api.get.call_args_list[1].kwargs['range'] == '100-104' + + +def test_iter_tickets_stops_at_updated_since(): + tickets = [ + {'id': 1, 'date_mod': '2024-02-01 10:00:00'}, + {'id': 2, 'date_mod': '2024-01-01 10:00:00'}, + {'id': 3, 'date_mod': '2023-12-01 10:00:00'}, + ] + + with _patched_api() as api: + api.get.return_value = _mock_response(tickets, '0-2/3') + collected = list(api.iter_tickets(updated_since='2024-01-15 00:00:00')) + + assert [ticket['id'] for ticket in collected] == [1] + assert api.get.call_args.kwargs['sort'] == _TICKET_SORT_DATE_MOD + assert api.get.call_args.kwargs['order'] == 'DESC' + + +def test_iter_tickets_created_after_stops_at_opening_date(): + tickets = [ + {'id': 1, 'date': '2024-02-01 10:00:00'}, + {'id': 2, 'date': '2024-01-01 10:00:00'}, + ] + + with _patched_api() as api: + api.get.return_value = _mock_response(tickets, '0-1/2') + collected = list(api.iter_tickets(created_after='2024-01-15 00:00:00')) + + assert [ticket['id'] for ticket in collected] == [1] + assert api.get.call_args.kwargs['sort'] == _TICKET_SORT_OPENING_DATE + + +def test_get_item_skips_zero_ids(): + users = [{'id': 1, 'name': 'user1'}] + + with _patched_api() as api: + api.get.return_value = _mock_response(users) + assert api.get_item('Location', 0) == {} + assert api.get_item('User', 1) == users[0] + + api.get.assert_called_once_with( + 'apirest.php', 'User', '1', + expand_dropdowns='false', get_hateoas='false', + ) + + +def test_get_item_returns_empty_for_missing_item(): + users = [{'id': 1, 'name': 'user1'}] + + def _get(*args, **kwargs): + if args[1] == 'Location': + response = MagicMock() + response.status = 404 + response.body = b'["ERROR_ITEM_NOT_FOUND","Item not found"]' + response.json = response.body + response.headers = {} + response.request = MagicMock() + raise APIException(response) + return _mock_response(users) + + with _patched_api() as api: + api.get.side_effect = _get + assert api.get_item('User', 1) == users[0] + assert api.get_item('Location', 999) == {} + + +def test_iter_tickets_incremental_uses_paginated_list(): + tickets = [ + {'id': 1, 'date_mod': '2024-02-01 10:00:00'}, + {'id': 2, 'date_mod': '2024-02-02 10:00:00'}, + ] + + with _patched_api() as api: + api.get.return_value = _mock_response(tickets, '0-1/2') + api.get_item = MagicMock() + collected = list(api.iter_tickets(updated_since='2024-01-01')) + + assert collected == tickets + assert api.get.call_args.args[1] == 'Ticket' + api.get_item.assert_not_called() + + +def test_iter_ticket_links_delegates_to_api_get(): + links = [{'id': 1, 'tickets_id_1': 10, 'tickets_id_2': 20}] + + with _patched_api() as api: + api._api_get.return_value = iter(links) + assert list(api.iter_ticket_links()) == links + + api._api_get.assert_called_once_with('Ticket_Ticket', expand_dropdowns='false') diff --git a/backend/python/plugins/glpi/tests/collection_test.py b/backend/python/plugins/glpi/tests/collection_test.py new file mode 100644 index 00000000000..e6294cce83f --- /dev/null +++ b/backend/python/plugins/glpi/tests/collection_test.py @@ -0,0 +1,225 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import json +from unittest.mock import MagicMock + +import pytest +from pydevlake.model import raw_data_params +from sqlmodel import Session + +from glpi.collection import ( + prepare_on_demand_raw, + referenced_group_ids, + referenced_user_ids, +) +from glpi.main import GlpiPlugin +from glpi.models import Ticket +from glpi.streams import ticket_substreams +from glpi.streams.ticket_substreams import TicketLinks, TicketLinksCollector +from glpi.streams.tickets import Tickets +from tests.common import GLPI_API_BASE_URL, collect_context, glpi_api_mock + + +def test_collect_normalizes_bare_user_fields(): + with glpi_api_mock() as api: + api.iter_tickets.return_value = iter([{ + 'id': 1, + 'date_mod': '2024-01-01', + 'users_id_recipient': 101, + 'users_id_lastupdater': 102, + }]) + api.get_item.side_effect = lambda itemtype, item_id: { + 'id': item_id, 'name': f'user{item_id}', + } + collected = [ + raw for raw, _ in Tickets(GlpiPlugin().name).collect({}, collect_context()) + ] + + assert collected[0]['users_id_recipient'] == 'user101 (101)' + assert collected[0]['users_id_lastupdater'] == 'user102 (102)' + + +def test_collect_skips_zero_catalog_id(): + with glpi_api_mock() as api: + api.iter_tickets.return_value = iter([{ + 'id': 1, + 'date_mod': '2024-01-01', + 'locations_id': 0, + 'requesttypes_id': 2, + }]) + api.get_item.side_effect = lambda itemtype, item_id: {'name': 'Email'} + collected = [ + raw for raw, _ in Tickets(GlpiPlugin().name).collect({}, collect_context()) + ] + + assert collected[0]['locations_id'] == 0 + assert collected[0]['requesttypes_id'] == 'Email (2)' + api.get_item.assert_called_once_with('RequestType', 2) + + +def test_collect_resolves_catalog_completename(): + with glpi_api_mock() as api: + api.iter_tickets.return_value = iter([{ + 'id': 1, + 'date_mod': '2024-01-01', + 'locations_id': 5, + }]) + api.get_item.side_effect = lambda itemtype, item_id: { + 'completename': 'Root > Building A', + 'name': 'Building A', + } + collected = [ + raw for raw, _ in Tickets(GlpiPlugin().name).collect({}, collect_context()) + ] + + assert collected[0]['locations_id'] == 'Root > Building A (5)' + api.get_item.assert_called_once_with('Location', 5) + + +def test_referenced_user_ids_from_raw_rows(): + plugin = GlpiPlugin() + ctx = collect_context() + + with Session(ctx.engine) as session: + _seed_raw(session, plugin, ctx, { + 'tickets': [{'id': 1, 'users_id_recipient': 'Requester (101)'}], + 'ticketusers': [{'id': 2, 'users_id': 'Tech (102)'}], + 'tickettasks': [{'id': 3, 'users_id_tech': 'Worker (103)'}], + }) + assert referenced_user_ids(session, plugin, ctx) == {101, 102, 103} + + +def _seed_raw(session, plugin, ctx, rows_by_stream): + params = raw_data_params(ctx.connection.id, ctx.scope.id) + for stream_name, rows in rows_by_stream.items(): + raw_model = plugin.get_stream(stream_name).raw_model(session) + for row in rows: + session.add(raw_model(params=params, data=json.dumps(row))) + session.commit() + + +def test_referenced_group_ids_from_raw_rows(): + plugin = GlpiPlugin() + ctx = collect_context() + + with Session(ctx.engine) as session: + _seed_raw(session, plugin, ctx, { + 'ticketgroups': [{'id': 1, 'groups_id': 'Support (5)'}], + }) + assert referenced_group_ids(session, plugin, ctx) == {5} + + +def test_collect_uses_project_time_after_when_not_incremental(): + from pydevlake.testing import ContextBuilder + from glpi.main import GlpiPlugin + + context = ( + ContextBuilder(GlpiPlugin()) + .with_connection( + endpoint=GLPI_API_BASE_URL, + token='test_app_token', + secret_key='test_user_token', + timezone='UTC', + ) + .with_scope('1', name='Test Entity') + .with_time_after(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc)) + .build() + ) + + with glpi_api_mock() as api: + api.iter_tickets.return_value = iter([{'id': 1, 'date_mod': '2024-06-02'}]) + list(Tickets(GlpiPlugin().name).collect({}, context)) + + api.iter_tickets.assert_called_once_with( + entity_id='1', + updated_since=None, + created_after='2024-01-01 00:00:00', + ) + + +@pytest.mark.parametrize('raw,ticket_ids,expected', [ + ({'tickets_id_1': 1, 'tickets_id_2': 2}, {1, 3}, True), + ({'tickets_id_1': 1, 'tickets_id_2': 2}, {3, 4}, False), + ({'tickets_id1': 5, 'tickets_id2': 6}, {6}, True), +]) +def test_link_involves_tickets(raw, ticket_ids, expected): + collector = TicketLinksCollector(TicketLinks(GlpiPlugin().name)) + context = collect_context(collect_ticket_links=True) + + session = MagicMock() + session.query.return_value.where.return_value.all.return_value = [ + Ticket(id=ticket_id, connection_id=context.connection.id) + for ticket_id in ticket_ids + ] + + with glpi_api_mock(ticket_substreams) as api: + api.iter_ticket_links.return_value = iter([raw]) + collected = list(collector.fetch({}, session, context)) + + if expected: + assert len(collected) == 1 + result_raw, _ = collected[0] + for key, value in raw.items(): + assert result_raw[key] == value + assert result_raw['x_request_url'] == f'{GLPI_API_BASE_URL}/apirest.php/Ticket_Ticket' + assert result_raw['x_request_input'] == {} + else: + assert collected == [] + + +def test_prepare_on_demand_raw_skips_empty_and_invalid_ids(): + assert prepare_on_demand_raw({}, 7, 'User', GLPI_API_BASE_URL) is None + assert prepare_on_demand_raw({'name': 'jdoe'}, 0, 'User', GLPI_API_BASE_URL) is None + + +def test_prepare_on_demand_raw_sets_id_and_url(): + prepared = prepare_on_demand_raw({'name': 'jdoe'}, 7, 'User', GLPI_API_BASE_URL) + assert prepared['id'] == 7 + assert prepared['x_request_url'] == f'{GLPI_API_BASE_URL}/apirest.php/User/7' + + +def test_ticket_links_collector_skips_when_disabled(): + collector = TicketLinksCollector(TicketLinks(GlpiPlugin().name)) + context = collect_context(collect_ticket_links=False) + + with glpi_api_mock(ticket_substreams) as api: + collected = list(collector.fetch({}, MagicMock(), context)) + + assert collected == [] + api.assert_not_called() + + +@pytest.mark.parametrize('stream_class,flag_name', [ + ('TicketFollowups', 'collect_followups'), + ('TicketTasks', 'collect_tasks'), + ('TicketLogs', 'collect_logs'), + ('TicketSolutions', 'collect_solutions'), + ('TicketUsers', 'collect_assignees'), + ('TicketGroups', 'collect_assignees'), +]) +def test_optional_substream_skips_when_disabled(stream_class, flag_name): + stream = getattr(ticket_substreams, stream_class)(GlpiPlugin().name) + context = collect_context(**{flag_name: False}) + mock_api = MagicMock() + mock_api.base_url = GLPI_API_BASE_URL + mock_api.ticket_subitems.return_value = iter([{'id': 2}]) + parent = Ticket(id=1, connection_id=context.connection.id) + + collected = list(stream.collect({}, context, parent, mock_api)) + + assert collected == [] + mock_api.ticket_subitems.assert_not_called() diff --git a/backend/python/plugins/glpi/tests/common.py b/backend/python/plugins/glpi/tests/common.py new file mode 100644 index 00000000000..78f880a17e9 --- /dev/null +++ b/backend/python/plugins/glpi/tests/common.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager +from types import ModuleType +from typing import Iterator +from unittest.mock import create_autospec, patch + +from pydevlake.model import RawModel +from pydevlake.stream import Stream +from pydevlake.testing import ContextBuilder +from sqlmodel import Session + +from glpi.api import GlpiAPI +from glpi.main import GlpiPlugin +from glpi.streams import tickets as tickets_stream + +GLPI_API_BASE_URL = 'http://glpi.example.com' + + +def ensure_stream_raw_table(engine, stream: Stream) -> None: + table_name = stream.raw_model_table + if table_name not in RawModel.metadata.tables: + with Session(engine) as session: + stream.raw_model(session) + return + RawModel.metadata.tables[table_name].create(engine, checkfirst=True) + + +def collect_context( + scope_id='1', + *, + scope_name='Test Entity', + scope_completename=None, + **scope_config_kwargs, +): + scope_kwargs = {'name': scope_name} + if scope_completename is not None: + scope_kwargs['completename'] = scope_completename + return ( + ContextBuilder(GlpiPlugin()) + .with_connection( + endpoint=GLPI_API_BASE_URL, + token='test_app_token', + secret_key='test_user_token', + timezone='UTC', + ) + .with_scope(scope_id, **scope_kwargs) + .with_scope_config(**scope_config_kwargs) + .build() + ) + + +@contextmanager +def glpi_api_mock(module: ModuleType = tickets_stream) -> Iterator[GlpiAPI]: + api = create_autospec(GlpiAPI, instance=True) + api.base_url = GLPI_API_BASE_URL + with patch.object(module, 'GlpiAPI') as mock_cls: + mock_cls.return_value.__enter__.return_value = api + yield api diff --git a/backend/python/plugins/glpi/tests/enrichment_test.py b/backend/python/plugins/glpi/tests/enrichment_test.py new file mode 100644 index 00000000000..096499f2620 --- /dev/null +++ b/backend/python/plugins/glpi/tests/enrichment_test.py @@ -0,0 +1,313 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import Optional + +import pytest +import pydevlake.domain_layer.ticket as ticket +from pydevlake.model import domain_id +from sqlmodel import Session + +from glpi.main import GlpiPlugin +from glpi.streams.ticket_domain import TicketDomain +from glpi.streams.tickets import Tickets +from glpi.streams.ticket_substreams import TicketLogs +from glpi.models import ( + Group, + ITILSolution, + Ticket, + TicketLink, + TicketLog, + TicketUser, + User, +) +from tests.common import collect_context, ensure_stream_raw_table + + +def test_resolve_glpi_user_ref_from_lookup(): + context = collect_context() + stream = Tickets(GlpiPlugin().name) + + User.__table__.create(context.engine, checkfirst=True) + with Session(context.engine) as session: + session.add(User(id=2, name='glpi', connection_id=context.connection.id)) + session.commit() + ensure_stream_raw_table(context.engine, stream) + list(stream.extractor.fetch({}, session, context)) + + ticket_model = stream.extract({'id': 1, 'users_id_recipient': 'glpi'}, context) + assert ticket_model.requester_name == 'glpi' + assert ticket_model.users_id_recipient == 2 + + +def _build_raw_ticket(props: dict) -> dict: + return { + 'id': 1, + 'name': 'Issue', + 'itilcategories_id': 'Network (99)', + 'users_id_recipient': 'Requester User (101)', + } | props + + +def test_apply_ticket_enrichment_with_bare_ids(): + context = collect_context( + '2', + scope_name='Systems', + scope_completename='Test > Systems', + ) + raw = _build_raw_ticket({'entities_id': 2}) + stream = Tickets(GlpiPlugin().name) + ticket_model = stream.extract(raw, context) + + assert ticket_model.entities_id == 'Test > Systems' + assert ticket_model.category_name == 'Network' + assert ticket_model.requester_name == 'Requester User' + + +def test_apply_ticket_enrichment_leaves_raw_unchanged(): + custom_fields_dict = {'custom_plugin_field': 'value'} + context = collect_context() + raw = _build_raw_ticket(custom_fields_dict) + raw_snapshot = dict(raw) + + stream = Tickets(GlpiPlugin().name) + ticket_model = stream.extract(raw, context) + + assert raw == raw_snapshot + assert ticket_model.category_name == 'Network' + assert ticket_model.itilcategories_id == 99 + assert ticket_model.requester_name == 'Requester User' + assert ticket_model.users_id_recipient == 101 + assert ticket_model.plugin_fields_json == json.dumps(custom_fields_dict) + + +def test_enrich_ticket_from_tool_records(): + context = collect_context() + connection_id = context.connection.id + stream = TicketDomain(GlpiPlugin().name) + + for model in (TicketUser, TicketLink, ITILSolution, *stream.domain_models): + model.__table__.create(context.engine, checkfirst=True) + + with Session(context.engine) as session: + session.add(TicketUser( + id=1, + connection_id=connection_id, + tickets_id=1, + type=2, + users_id=101, + user_name='Tech User', + )) + session.add(ITILSolution( + id=1, + connection_id=connection_id, + tickets_id=1, + content='

Fixed.

', + )) + session.commit() + + ticket_model = Ticket(id=1, name='Issue', connection_id=connection_id) + stream.convertor.process(ticket_model, session, context) + + assert ticket_model.assignee_id == 101 + assert ticket_model.assignee_name == 'Tech User' + assert ticket_model.solution_content == '

Fixed.

' + + + +def _parent_ticket_ids_from_link( + context, + *, + link_type: int, + tickets_id_1: int, + tickets_id_2: int, + ticket_ids: list[int], +) -> dict[int, Optional[int]]: + connection_id = context.connection.id + stream = TicketDomain(GlpiPlugin().name) + for model in (TicketUser, TicketLink, ITILSolution, *stream.domain_models): + model.__table__.create(context.engine, checkfirst=True) + + parent_ids: dict[int, Optional[int]] = {} + with Session(context.engine) as session: + session.add(TicketLink( + id=1, + connection_id=connection_id, + link=link_type, + tickets_id_1=tickets_id_1, + tickets_id_2=tickets_id_2, + )) + session.commit() + for ticket_id in ticket_ids: + ticket_model = Ticket(id=ticket_id, name='Issue', connection_id=connection_id) + stream.convertor.process(ticket_model, session, context) + parent_ids[ticket_id] = ticket_model.parent_ticket_id + return parent_ids + + +@pytest.mark.parametrize('link_type,tickets_id_1,tickets_id_2,ticket_id,expected', [ + (3, 1, 10, 1, 10), + (3, 1, 10, 10, None), + (4, 10, 1, 1, 10), + (4, 10, 1, 10, None), +]) +def test_parent_ticket_id_from_links(link_type, tickets_id_1, tickets_id_2, ticket_id, expected): + context = collect_context() + parent_ids = _parent_ticket_ids_from_link( + context, + link_type=link_type, + tickets_id_1=tickets_id_1, + tickets_id_2=tickets_id_2, + ticket_ids=[ticket_id], + ) + assert parent_ids[ticket_id] == expected + + +@pytest.mark.parametrize('old_value,new_value,expected_from,expected_to', [ + ('', 'New', None, 'TODO'), + ('', 'Solved', None, 'DONE'), + ('', 'Processing (assigned)', None, 'IN_PROGRESS'), + ('', '2', None, 'IN_PROGRESS'), + ('', '5', None, 'DONE'), + ('2', '3', 'IN_PROGRESS', 'IN_PROGRESS'), +]) +def test_ticket_log_status_changelog(old_value, new_value, expected_from, expected_to): + context = collect_context() + log = TicketLog( + id=1, + tickets_id=1, + linked_action=0, + id_search_option=12, + old_value=old_value, + new_value=new_value, + ) + changelogs = list(TicketLogs(GlpiPlugin().name).convert(log, context)) + assert len(changelogs) == 1 + changelog = changelogs[0] + assert changelog.field_name == 'status' + assert changelog.from_value == expected_from + assert changelog.to_value == expected_to + assert changelog.original_from_value == (old_value or None) + assert changelog.original_to_value == new_value + + +def test_ticket_log_assignee_user_relation(): + context = collect_context() + log = TicketLog( + id=6, + tickets_id=1, + linked_action=15, + itemtype_link='User', + id_search_option=5, + old_value='', + new_value='glpi (2)', + ) + changelogs = list(TicketLogs(GlpiPlugin().name).convert(log, context)) + assert len(changelogs) == 1 + changelog = changelogs[0] + assert changelog.field_name == 'assignee' + assert changelog.to_value == domain_id(User, context.connection.id, 2) + + +def test_ticket_log_assignee_group_relation(): + context = collect_context() + log = TicketLog( + id=6, + tickets_id=1, + linked_action=15, + itemtype_link='Group', + old_value='', + new_value='Support Team (5)', + ) + changelogs = list(TicketLogs(GlpiPlugin().name).convert(log, context)) + assert len(changelogs) == 1 + changelog = changelogs[0] + assert changelog.field_name == 'assignee' + assert changelog.to_value == domain_id(Group, context.connection.id, 5) + + +def test_ticket_log_assignee_field_change(): + context = collect_context() + log = TicketLog( + id=7, + tickets_id=1, + linked_action=0, + id_search_option=5, + old_value='', + new_value='Jane Smith (103)', + ) + changelogs = list(TicketLogs(GlpiPlugin().name).convert(log, context)) + assert len(changelogs) == 1 + changelog = changelogs[0] + assert changelog.field_name == 'assignee' + assert changelog.from_value is None + assert changelog.to_value == domain_id(User, context.connection.id, 103) + assert changelog.original_to_value == '103' + + +def test_ticket_log_requester_field_change(): + context = collect_context() + log = TicketLog( + id=104, + tickets_id=1, + linked_action=0, + id_search_option=4, + old_value='', + new_value='Jane Smith (103)', + ) + changelogs = list(TicketLogs(GlpiPlugin().name).convert(log, context)) + assert len(changelogs) == 1 + changelog = changelogs[0] + assert changelog.field_name == 'requester' + assert changelog.from_value is None + assert changelog.to_value == domain_id(User, context.connection.id, 103) + assert changelog.original_to_value == '103' + + +def test_ticket_log_priority_change(): + context = collect_context() + log = TicketLog( + id=4, + tickets_id=1, + linked_action=0, + id_search_option=3, + old_value='3', + new_value='High', + ) + changelogs = list(TicketLogs(GlpiPlugin().name).convert(log, context)) + assert len(changelogs) == 1 + changelog = changelogs[0] + assert changelog.field_name == 'priority' + assert changelog.from_value == 'Medium' + assert changelog.to_value == 'High' + + +def test_ticket_log_type_change(): + context = collect_context() + log = TicketLog( + id=5, + tickets_id=1, + linked_action=0, + id_search_option=14, + old_value='Incident', + new_value='Request', + ) + changelogs = list(TicketLogs(GlpiPlugin().name).convert(log, context)) + assert len(changelogs) == 1 + changelog = changelogs[0] + assert changelog.field_name == 'type' + assert changelog.from_value == 'INCIDENT' + assert changelog.to_value == 'TASK' diff --git a/backend/python/plugins/glpi/tests/parsing_test.py b/backend/python/plugins/glpi/tests/parsing_test.py new file mode 100644 index 00000000000..7be23b8554b --- /dev/null +++ b/backend/python/plugins/glpi/tests/parsing_test.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from glpi.parsing import on_demand_id_from_url, parse_glpi_dropdown, parse_glpi_user + + +def test_parse_glpi_dropdown_with_name_and_id(): + name, user_id = parse_glpi_dropdown('Jane Smith (103)') + assert name == 'Jane Smith' + assert user_id == 103 + + +def test_parse_glpi_dropdown_with_numeric_id(): + name, user_id = parse_glpi_dropdown(101) + assert name is None + assert user_id == 101 + + +def test_parse_glpi_user_with_id(): + name, user_id = parse_glpi_user('John Doe (101)') + assert name == 'John Doe' + assert user_id == '101' + + +def test_parse_glpi_user_without_id(): + name, user_id = parse_glpi_user('John Doe') + assert name == 'John Doe' + assert user_id is None + + +def test_on_demand_id_from_url_uses_raw_id(): + assert on_demand_id_from_url({'id': 7}, 'User') == 7 + + +def test_on_demand_id_from_url_parses_request_url(): + raw = {'x_request_url': 'http://glpi.example.com/apirest.php/User/7'} + assert on_demand_id_from_url(raw, 'User') == 7 diff --git a/backend/python/plugins/glpi/tests/plugin_test.py b/backend/python/plugins/glpi/tests/plugin_test.py new file mode 100644 index 00000000000..56b8a46b24e --- /dev/null +++ b/backend/python/plugins/glpi/tests/plugin_test.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest + +from pydevlake.testing import assert_valid_plugin, assert_plugin_run + +from glpi.main import GlpiPlugin +from glpi.models import Connection, ScopeConfig + + +def test_valid_plugin(): + assert_valid_plugin(GlpiPlugin()) + + +def test_valid_plugin_and_connection(): + endpoint = os.environ.get('GLPI_ENDPOINT') + token = os.environ.get('GLPI_TOKEN') + secret_key = os.environ.get('GLPI_SECRET_KEY') + if not (endpoint and token and secret_key): + pytest.skip("GLPI_ENDPOINT, GLPI_TOKEN, and GLPI_SECRET_KEY env vars required") + + connection = Connection( + id=1, + name='test_connection', + endpoint=endpoint, + token=token, + secret_key=secret_key, + timezone='UTC', + ) + scope_config = ScopeConfig(id=1, name='test_config') + assert_plugin_run(GlpiPlugin(), connection, scope_config) diff --git a/backend/python/plugins/glpi/tests/remote_scopes_test.py b/backend/python/plugins/glpi/tests/remote_scopes_test.py new file mode 100644 index 00000000000..b58e5689786 --- /dev/null +++ b/backend/python/plugins/glpi/tests/remote_scopes_test.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glpi.main as glpi_main +from pydevlake.message import RemoteScope + +from glpi.main import GlpiPlugin, ROOT_GROUP_ID +from tests.common import collect_context, glpi_api_mock + + +def test_remote_scopes_returns_all_entities_sorted(): + plugin = GlpiPlugin() + with glpi_api_mock(glpi_main) as api: + api.entities.return_value = [ + {'id': 2, 'name': 'Y', 'completename': 'Root entity > X > Y', 'entities_id': 1}, + {'id': 0, 'name': 'Root entity', 'completename': 'Root entity', 'entities_id': 0}, + {'id': 1, 'name': 'X', 'completename': 'Root entity > X', 'entities_id': 0}, + ] + entities = list(plugin.remote_scopes(collect_context().connection, ROOT_GROUP_ID)) + + assert [entity.completename for entity in entities] == [ + 'Root entity', + 'Root entity > X', + 'Root entity > X > Y', + ] + + +def test_make_remote_scopes_under_root_group(): + plugin = GlpiPlugin() + with glpi_api_mock(glpi_main) as api: + api.entities.return_value = [ + {'id': 0, 'name': 'Root entity', 'completename': 'Root entity', 'entities_id': 0}, + {'id': 1, 'name': 'X', 'completename': 'Root entity > X', 'entities_id': 0}, + ] + scopes = list(plugin.make_remote_scopes(collect_context().connection, ROOT_GROUP_ID)) + + assert len(scopes) == 2 + assert all(isinstance(scope, RemoteScope) for scope in scopes) + assert all(scope.parent_id == ROOT_GROUP_ID for scope in scopes) + assert [scope.name for scope in scopes] == ['Root entity', 'X'] diff --git a/backend/python/plugins/glpi/tests/streams_test.py b/backend/python/plugins/glpi/tests/streams_test.py new file mode 100644 index 00000000000..db94ab65f8b --- /dev/null +++ b/backend/python/plugins/glpi/tests/streams_test.py @@ -0,0 +1,463 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +import pytest + +import pydevlake.domain_layer.crossdomain as crossdomain +import pydevlake.domain_layer.ticket as ticket +from pydevlake.model import domain_id +from pydevlake.testing import assert_stream_convert + +from glpi.main import GlpiPlugin +from glpi.models import ( + Followup, + Group, + Ticket, + TicketGroup, + TicketLink, + TicketLog, + TicketTask, + TicketUser, + User, +) +from glpi.streams.ticket_domain import TicketDomain +from tests.common import GLPI_API_BASE_URL, collect_context + + +@pytest.fixture +def context(): + return collect_context(scope_completename='Root > Test Entity') + + +def test_tickets_stream(context): + ticket_model = Ticket( + id=1, + name='Network connectivity issue', + content='

Cannot connect to the server.

', + date=datetime.datetime(2024, 1, 15, 8, 0, 0), + date_creation=datetime.datetime(2024, 1, 15, 10, 0, 0), + date_mod=datetime.datetime(2024, 1, 16, 12, 0, 0), + solvedate=datetime.datetime(2024, 1, 17, 9, 0, 0), + status=5, + type=1, + urgency=3, + impact=3, + priority=4, + entities_id='1', + itilcategories_id=99, + category_name='Network', + users_id_recipient=101, + requester_name='Requester User', + assignee_id=101, + assignee_name='Tech User', + parent_ticket_id=10, + solution_content='

Replaced cable.

', + time_to_resolve=datetime.datetime(2024, 1, 20, 17, 0, 0), + actiontime=3600, + solve_delay_stat=7200, + takeintoaccount_delay_stat=600, + connection_id=context.connection.id, + ) + + issue = ticket.Issue( + url=f'{GLPI_API_BASE_URL}/front/ticket.form.php?id=1', + issue_key='1', + title='Network connectivity issue', + description='

Cannot connect to the server.

\n\n--- Solution ---\n

Replaced cable.

', + type='INCIDENT', + original_type='Incident', + status='DONE', + original_status='Solved', + created_date='2024-01-15 08:00:00', + updated_date='2024-01-16 12:00:00', + resolution_date='2024-01-17 09:00:00', + due_date='2024-01-20 17:00:00', + creator_id=domain_id(User, context.connection.id, 101), + creator_name='Requester User', + assignee_id=domain_id(User, context.connection.id, 101), + assignee_name='Tech User', + parent_issue_id=domain_id(Ticket, context.connection.id, 10), + is_subtask=True, + priority='High', + urgency='Medium', + severity='Medium', + component='Network', + original_project='1', + lead_time_minutes=120, + ) + issue_id = domain_id(Ticket, context.connection.id, 1) + domain_models = list(TicketDomain(GlpiPlugin().name).convert(ticket_model, context)) + assert domain_models == [ + issue, + ticket.BoardIssue(board_id=context.scope.domain_id(), issue_id=None), + ticket.IssueLabel(issue_id=issue_id, label_name='Network'), + ] + + +def test_tickets_stream_extract_returns_tool_model(context): + raw = { + 'id': 1, + 'name': 'Network connectivity issue', + 'entities_id': 'Root > Test Entity (1)', + 'itilcategories_id': 'Network (99)', + 'users_id_recipient': 'Requester User (101)', + } + stream = GlpiPlugin().get_stream('tickets') + tool_model = stream.extract(raw, context) + assert tool_model is not None + assert tool_model.id == 1 + assert tool_model.category_name == 'Network' + assert tool_model.requester_name == 'Requester User' + + +def test_tickets_stream_new_status(context): + raw = { + 'id': 10, + 'name': 'Printer not working', + 'content': None, + 'date': '2024-02-01 09:00:00', + 'date_creation': None, + 'date_mod': '2024-02-01 09:30:00', + 'solvedate': None, + 'closedate': None, + 'status': 1, + 'type': 2, + 'urgency': 1, + 'impact': 1, + 'priority': 1, + 'entities_id': '2', + 'itilcategories_id': None, + 'users_id_recipient': None, + 'users_id_lastupdater': None, + 'requesttypes_id': None, + 'time_to_resolve': None, + 'actiontime': None, + 'solve_delay_stat': None, + 'takeintoaccount_delay_stat': None, + 'x_request_url': f'{GLPI_API_BASE_URL}/apirest.php/Ticket', + 'x_request_input': {'EntityId': '1'}, + } + + issue = ticket.Issue( + url=f'{GLPI_API_BASE_URL}/front/ticket.form.php?id=10', + issue_key='10', + title='Printer not working', + description=None, + type='TASK', + original_type='Request', + status='TODO', + original_status='New', + created_date='2024-02-01 09:00:00', + updated_date='2024-02-01 09:30:00', + resolution_date=None, + due_date=None, + creator_id=None, + creator_name=None, + assignee_id=None, + assignee_name=None, + parent_issue_id=None, + priority='Very Low', + urgency='Very Low', + severity='Very Low', + component=None, + original_project='2', + time_spent_minutes=None, + lead_time_minutes=None, + ) + expected = [issue, ticket.BoardIssue(board_id=context.scope.domain_id(), issue_id=issue.id)] + assert_stream_convert(GlpiPlugin, 'ticketdomain', raw, expected, context) + + +@pytest.mark.parametrize('stream_name,tool_model,assignee_model,assignee_id,assignee_name', [ + ( + 'ticketusers', + TicketUser(id=10, tickets_id=1, type=2, users_id=101, user_name='Tech User'), + User, 101, 'Tech User', + ), + ( + 'ticketgroups', + TicketGroup(id=11, tickets_id=1, type=2, groups_id=5, group_name='Support Team'), + Group, 5, 'Support Team', + ), +]) +def test_ticket_assignees_from_subitems( + context, stream_name, tool_model, assignee_model, assignee_id, assignee_name, +): + tool_model.connection_id = context.connection.id + expected = ticket.IssueAssignee( + issue_id=domain_id(Ticket, context.connection.id, 1), + assignee_id=domain_id(assignee_model, context.connection.id, assignee_id), + assignee_name=assignee_name, + ) + assert _convert_subitem(stream_name, tool_model, context) == [expected] + + +def _convert_subitem(stream_name, tool_model, context): + stream = GlpiPlugin().get_stream(stream_name) + return list(stream.convert(tool_model, context)) + + +def test_ticket_followups_from_subitems(context): + tool_model = Followup( + id=5, + items_id=1, + users_id=103, + content='Investigated the issue, awaiting vendor response.', + date=datetime.datetime(2024, 1, 16, 9, 0, 0), + date_mod=datetime.datetime(2024, 1, 16, 9, 30, 0), + is_private=0, + connection_id=context.connection.id, + ) + results = _convert_subitem('ticketfollowups', tool_model, context) + assert len(results) == 1 + assert results[0].issue_id == domain_id(Ticket, context.connection.id, 1) + assert results[0].body == 'Investigated the issue, awaiting vendor response.' + assert results[0].account_id == domain_id(User, context.connection.id, 103) + assert results[0].created_date == datetime.datetime(2024, 1, 16, 9, 0) + assert results[0].updated_date == datetime.datetime(2024, 1, 16, 9, 30) + + +def test_ticket_worklogs_from_subitems(context): + tool_model = TicketTask( + id=7, + tickets_id=1, + users_id=101, + content='Replaced the network cable.', + begin=datetime.datetime(2024, 1, 17, 8, 0, 0), + end=datetime.datetime(2024, 1, 17, 9, 0, 0), + actiontime=3600, + connection_id=context.connection.id, + ) + results = _convert_subitem('tickettasks', tool_model, context) + assert len(results) == 1 + assert results[0].issue_id == domain_id(Ticket, context.connection.id, 1) + assert results[0].author_id == domain_id(User, context.connection.id, 101) + assert results[0].comment == 'Replaced the network cable.' + assert results[0].time_spent_minutes == 60 + assert results[0].started_date == datetime.datetime(2024, 1, 17, 8, 0) + assert results[0].logged_date == datetime.datetime(2024, 1, 17, 8, 0) + + +def test_ticket_changelogs_from_subitems_smoke(context): + tool_model = TicketLog( + id=100, + tickets_id=1, + itemtype='Ticket', + itemtype_link='', + linked_action=0, + user_name='admin (1)', + date_mod=datetime.datetime(2024, 1, 16, 10, 0, 0), + id_search_option=12, + old_value='New', + new_value='Processing (assigned)', + connection_id=context.connection.id, + ) + expected = ticket.IssueChangelogs( + id=domain_id(TicketLog, context.connection.id, 100), + issue_id=domain_id(Ticket, context.connection.id, 1), + author_id=domain_id(User, context.connection.id, 1), + author_name='admin', + field_id='12', + field_name='status', + original_from_value='New', + original_to_value='Processing (assigned)', + from_value='TODO', + to_value='IN_PROGRESS', + created_date=datetime.datetime(2024, 1, 16, 10, 0), + ) + results = _convert_subitem('ticketlogs', tool_model, context) + assert len(results) == 1 + assert results[0] == expected + + +@pytest.mark.parametrize('tool_model', [ + TicketLog( + id=103, + tickets_id=1, + itemtype='Ticket', + itemtype_link='User', + linked_action=15, + user_name='admin (1)', + date_mod=datetime.datetime(2024, 1, 16, 11, 30, 0), + id_search_option=0, + old_value='', + new_value='Jane Smith (103)', + ), + TicketLog( + id=102, + tickets_id=1, + itemtype='Ticket', + itemtype_link='ITILFollowup', + linked_action=17, + user_name='admin (1)', + date_mod=datetime.datetime(2024, 1, 16, 12, 0, 0), + id_search_option=0, + old_value='', + new_value='Added follow-up', + ), +]) +def test_ticket_changelogs_skips_non_changelog_entries(context, tool_model): + tool_model.connection_id = context.connection.id + assert _convert_subitem('ticketlogs', tool_model, context) == [] + + +def test_ticket_links_from_subitems(context): + tool_model = TicketLink( + id=3, + tickets_id=1, + tickets_id_1=1, + tickets_id_2=99, + link=1, + connection_id=context.connection.id, + ) + expected = ticket.IssueRelationship( + source_issue_id=domain_id(Ticket, context.connection.id, 1), + target_issue_id=domain_id(Ticket, context.connection.id, 99), + original_type='linked', + ) + assert _convert_subitem('ticketlinks', tool_model, context) == [expected] + + +def test_groups_stream(context): + raw = { + 'id': 5, + 'name': 'Support', + 'completename': 'Root > Support', + 'x_request_url': f'{GLPI_API_BASE_URL}/apirest.php/Group', + } + plugin = GlpiPlugin() + stream = plugin.get_stream('groups') + tool_model = stream.extract(raw) + tool_model.connection_id = context.connection.id + account = list(stream.convert(tool_model, context))[0] + assert account.full_name == 'Root > Support' + assert account.user_name == 'Support' + assert account.email == 'group-5@glpi.local' + + +def test_users_stream(context): + raw = { + 'id': 7, + 'name': 'jdoe', + 'firstname': 'John', + 'realname': 'Doe', + 'email': 'john@example.com', + 'x_request_url': f'{GLPI_API_BASE_URL}/apirest.php/User', + } + + expected = crossdomain.Account( + email='john@example.com', + full_name='John Doe', + user_name='jdoe', + avatar_url='', + organization='', + created_date='2024-01-01 00:00:00', + status=1, + ) + plugin = GlpiPlugin() + stream = plugin.get_stream('users') + tool_model = stream.extract(raw) + tool_model.connection_id = context.connection.id + results = list(stream.convert(tool_model, context)) + account = results[0] + assert account.email == expected.email + assert account.full_name == expected.full_name + assert account.user_name == expected.user_name + assert account.status == expected.status + + +def test_users_stream_extracts_id_from_request_url(context): + raw = { + 'x_request_url': f'{GLPI_API_BASE_URL}/apirest.php/User/7', + } + plugin = GlpiPlugin() + stream = plugin.get_stream('users') + tool_model = stream.extract(raw) + assert tool_model.id == 7 + + +def test_ticket_worklogs_skip_entries_without_time(context): + tool_model = TicketTask( + id=8, + tickets_id=15, + users_id=None, + content=None, + begin=None, + end=None, + actiontime=None, + connection_id=context.connection.id, + ) + assert _convert_subitem('tickettasks', tool_model, context) == [] + + +def test_tickets_stream_converts_timezone(context): + context.connection.timezone = 'America/Sao_Paulo' + raw = { + 'id': 99, + 'name': 'Timezone test', + 'content': None, + 'date': '2026-06-02 00:00:00', + 'date_creation': None, + 'date_mod': '2026-06-02 01:00:00', + 'solvedate': None, + 'closedate': None, + 'status': 1, + 'type': 1, + 'urgency': 3, + 'impact': 3, + 'priority': 3, + 'entities_id': '1', + 'itilcategories_id': None, + 'users_id_recipient': None, + 'users_id_lastupdater': None, + 'requesttypes_id': None, + 'time_to_resolve': None, + 'actiontime': None, + 'solve_delay_stat': None, + 'takeintoaccount_delay_stat': None, + 'x_request_url': f'{GLPI_API_BASE_URL}/apirest.php/Ticket', + 'x_request_input': {'EntityId': '1'}, + } + + issue = ticket.Issue( + url=f'{GLPI_API_BASE_URL}/front/ticket.form.php?id=99', + issue_key='99', + title='Timezone test', + description=None, + type='INCIDENT', + original_type='Incident', + status='TODO', + original_status='New', + created_date='2026-06-02 03:00:00', + updated_date='2026-06-02 04:00:00', + resolution_date=None, + due_date=None, + creator_id=None, + creator_name=None, + assignee_id=None, + assignee_name=None, + parent_issue_id=None, + priority='Medium', + urgency='Medium', + severity='Medium', + component=None, + original_project='1', + time_spent_minutes=None, + lead_time_minutes=None, + ) + expected = [issue, ticket.BoardIssue(board_id=context.scope.domain_id(), issue_id=issue.id)] + assert_stream_convert(GlpiPlugin, 'ticketdomain', raw, expected, context) diff --git a/backend/python/plugins/glpi/tests/timezone_test.py b/backend/python/plugins/glpi/tests/timezone_test.py new file mode 100644 index 00000000000..6209dec3678 --- /dev/null +++ b/backend/python/plugins/glpi/tests/timezone_test.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from glpi.timezone import connection_timezone, glpi_search_time_from_iso + + +def test_glpi_search_time_from_iso_converts_to_connection_timezone(): + tz = connection_timezone('Europe/Paris') + assert glpi_search_time_from_iso('2024-06-01T00:00:00Z', tz) == '2024-06-01 02:00:00' diff --git a/backend/python/pydevlake/pydevlake/stream.py b/backend/python/pydevlake/pydevlake/stream.py index 6534aadb8e3..b376810285e 100644 --- a/backend/python/pydevlake/pydevlake/stream.py +++ b/backend/python/pydevlake/pydevlake/stream.py @@ -22,6 +22,8 @@ from pydevlake.model import RawModel, ToolModel, ToolScope, DomainModel, DomainType from pydevlake.extractor import autoextract +_raw_model_classes: dict[str, type] = {} + class Stream: def __init__(self, plugin_name: str): @@ -58,26 +60,38 @@ def domain_models(self) -> Optional[list[Type[DomainModel]]]: pass def raw_model(self, session) -> Type[RawModel]: + model = self._get_raw_model_class() + table = RawModel.metadata.tables[self.raw_model_table] + table.create(session.get_bind(), checkfirst=True) + return model + + def _get_raw_model_class(self) -> Type[RawModel]: if self._raw_model is not None: return self._raw_model table_name = self.raw_model_table + cached = _raw_model_classes.get(table_name) + if cached is not None: + self._raw_model = cached + return cached - # Look for existing raw model for mapper in RawModel._sa_registry.mappers: model = mapper.class_ - if model.__tablename__ == table_name: + if getattr(model, '__tablename__', None) == table_name: + _raw_model_classes[table_name] = model self._raw_model = model - return self._raw_model + return model + + table_args = {'extend_existing': True} if table_name in RawModel.metadata.tables else None - # Create raw model class StreamRawModel(RawModel, table=True): __tablename__ = table_name + if table_args is not None: + __table_args__ = table_args + _raw_model_classes[table_name] = StreamRawModel self._raw_model = StreamRawModel - table = RawModel.metadata.tables[table_name] - table.create(session.get_bind(), checkfirst=True) - return self._raw_model + return StreamRawModel @property def raw_model_table(self): diff --git a/config-ui/src/plugins/components/connection-form/fields/index.tsx b/config-ui/src/plugins/components/connection-form/fields/index.tsx index cb55f5751d8..42650e71930 100644 --- a/config-ui/src/plugins/components/connection-form/fields/index.tsx +++ b/config-ui/src/plugins/components/connection-form/fields/index.tsx @@ -25,6 +25,7 @@ import { ConnectionProxy } from './proxy'; import { ConnectionRateLimit } from './rate-limit'; import { ConnectionAppId } from './app-id'; import { ConnectionSecretKey } from './secret-key'; +import { ConnectionTimezone } from './timezone'; interface Props { type: 'create' | 'update'; @@ -86,6 +87,8 @@ export const Form = ({ type, name, fields, initialValues, values, errors, setVal return ; case 'rateLimitPerHour': return ; + case 'timezone': + return ; default: return null; } diff --git a/config-ui/src/plugins/components/connection-form/fields/timezone.tsx b/config-ui/src/plugins/components/connection-form/fields/timezone.tsx new file mode 100644 index 00000000000..d72d78f799f --- /dev/null +++ b/config-ui/src/plugins/components/connection-form/fields/timezone.tsx @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo } from 'react'; +import { Button, Flex, Select } from 'antd'; + +import { Block } from '@/components'; +import { buildTimezoneOptions, formatTimezoneLabel, getBrowserTimezone } from '@/utils/timezones'; + +interface Props { + initialValue: string; + value: string; + setValue: (value: string) => void; + setError: (value: string | undefined) => void; + subLabel?: string; + placeholder?: string; + required?: boolean; +} + +export const ConnectionTimezone = ({ + initialValue, + value, + setValue, + setError, + subLabel, + placeholder, + required = false, +}: Props) => { + const browserTimezone = useMemo(() => getBrowserTimezone(), []); + const options = useMemo(() => buildTimezoneOptions(value), [value]); + + useEffect(() => { + setValue(initialValue); + }, [initialValue]); + + useEffect(() => { + if (required && !value?.trim()) { + setError('Timezone is required'); + return; + } + setError(undefined); + }, [value, required]); + + return ( + + + } + placeholder="Search entities by name or path…" + value={query} + onChange={(e) => setQuery(e.target.value)} + /> + {loading ? ( + + ) : ( +
+ + + + + + + + {filteredItems.map((item) => { + const checked = selectedItems.some((it) => it.id === item.id); + const disabled = disabledIds.has(item.id); + return ( + + + + + ); + })} + +
+ Entity
+ { + if (e.target.checked) { + onChangeSelectedItems([...selectedItems, item]); + } else { + onChangeSelectedItems(selectedItems.filter((it) => it.id !== item.id)); + } + }} + /> + {entityLabel(item)}
+ {!filteredItems.length &&
No entities found
} +
+ )} +
+ + ); +}; diff --git a/config-ui/src/plugins/register/glpi/index.ts b/config-ui/src/plugins/register/glpi/index.ts new file mode 100644 index 00000000000..5f16858cbe4 --- /dev/null +++ b/config-ui/src/plugins/register/glpi/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; +export * from './transformation'; diff --git a/config-ui/src/plugins/register/glpi/transformation.tsx b/config-ui/src/plugins/register/glpi/transformation.tsx new file mode 100644 index 00000000000..21ae8d2bbab --- /dev/null +++ b/config-ui/src/plugins/register/glpi/transformation.tsx @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Checkbox, Tag } from 'antd'; + +import { Block } from '@/components'; + +interface Props { + entities: string[]; + transformation: any; + setTransformation: React.Dispatch>; +} + +const OPTIONAL_COLLECT_FIELDS = [ + { + key: 'collectAssignees', + label: 'Assignees', + description: 'Assigned users and groups on each ticket', + defaultEnabled: true, + }, + { + key: 'collectFollowups', + label: 'Follow-ups', + description: 'ITIL follow-up comments as issue comments', + defaultEnabled: false, + }, + { + key: 'collectTasks', + label: 'Tasks', + description: 'Ticket tasks as issue worklogs', + defaultEnabled: false, + }, + { + key: 'collectLogs', + label: 'Change logs', + description: 'Ticket change history as issue changelogs', + defaultEnabled: false, + }, + { + key: 'collectSolutions', + label: 'Solutions', + description: 'Solution text on the issue record', + defaultEnabled: false, + }, + { + key: 'collectTicketLinks', + label: 'Ticket links', + description: 'Parent/child and related ticket links as issue relationships', + defaultEnabled: false, + }, +] as const; + +const isFieldEnabled = (transformation: any, key: string, defaultEnabled: boolean) => + transformation[key] ?? defaultEnabled; + +export const GlpiTransformation = ({ entities, transformation, setTransformation }: Props) => { + const setOptionalCollect = (enabled: boolean) => { + setTransformation({ + ...transformation, + ...Object.fromEntries(OPTIONAL_COLLECT_FIELDS.map(({ key }) => [key, enabled])), + }); + }; + + const allOptionalEnabled = OPTIONAL_COLLECT_FIELDS.every(({ key, defaultEnabled }) => + isFieldEnabled(transformation, key, defaultEnabled), + ); + + const allOptionalDisabled = OPTIONAL_COLLECT_FIELDS.every( + ({ key, defaultEnabled }) => !isFieldEnabled(transformation, key, defaultEnabled), + ); + + if (!entities.includes('TICKET')) { + return null; + } + + return ( + +
+ setOptionalCollect(true)} + > + Enable all + + setOptionalCollect(false)} + > + Disable all + +
+
+ {OPTIONAL_COLLECT_FIELDS.map(({ key, label, description, defaultEnabled }) => ( +
+ + setTransformation({ + ...transformation, + [key]: e.target.checked, + }) + } + > + {label} + +
{description}
+
+ ))} +
+
+ ); +}; diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 56c884ad0a4..262fe33b875 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -31,6 +31,7 @@ import { CursorConfig } from './cursor'; import { GitHubConfig } from './github'; import { GhCopilotConfig } from './gh-copilot'; import { GitLabConfig } from './gitlab'; +import { GlpiConfig } from './glpi'; import { GrafanaIrmConfig } from './grafana_irm'; import { IncidentioConfig } from './incidentio'; import { JenkinsConfig } from './jenkins'; @@ -66,6 +67,7 @@ export const pluginConfigs: IPluginConfig[] = [ GitHubConfig, GhCopilotConfig, GitLabConfig, + GlpiConfig, GrafanaIrmConfig, IncidentioConfig, JenkinsConfig, diff --git a/config-ui/src/utils/timezones.ts b/config-ui/src/utils/timezones.ts new file mode 100644 index 00000000000..fd2bd9d48f4 --- /dev/null +++ b/config-ui/src/utils/timezones.ts @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export interface TimezoneOption { + value: string; + label: string; +} + +let cachedOptions: TimezoneOption[] | null = null; + +export function getBrowserTimezone(): string | undefined { + try { + return Intl.DateTimeFormat().resolvedOptions().timeZone; + } catch { + return undefined; + } +} + +function getUtcOffsetMinutes(timezone: string, date = new Date()): number { + try { + const utcDate = new Date(date.toLocaleString('en-US', { timeZone: 'UTC' })); + const tzDate = new Date(date.toLocaleString('en-US', { timeZone: timezone })); + return (tzDate.getTime() - utcDate.getTime()) / 60_000; + } catch { + return 0; + } +} + +function sortTimezones(timezones: string[]): string[] { + return [...timezones].sort((left, right) => { + const offsetDiff = getUtcOffsetMinutes(left) - getUtcOffsetMinutes(right); + if (offsetDiff !== 0) { + return offsetDiff; + } + return left.localeCompare(right); + }); +} + +function listIanaTimezones(): string[] { + if ('supportedValuesOf' in Intl && typeof Intl.supportedValuesOf === 'function') { + return sortTimezones(Intl.supportedValuesOf('timeZone')); + } + + const fallback = new Set(['UTC']); + const browserTimezone = getBrowserTimezone(); + if (browserTimezone) { + fallback.add(browserTimezone); + } + return sortTimezones([...fallback]); +} + +export function formatTimezoneLabel(timezone: string): string { + try { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: timezone, + timeZoneName: 'shortOffset', + }).formatToParts(new Date()); + const offset = parts.find((part) => part.type === 'timeZoneName')?.value ?? ''; + const name = timezone.replace(/_/g, ' '); + return offset ? `${name} (${offset})` : name; + } catch { + return timezone.replace(/_/g, ' '); + } +} + +export function getTimezoneOptions(): TimezoneOption[] { + if (!cachedOptions) { + cachedOptions = listIanaTimezones().map((timezone) => ({ + value: timezone, + label: formatTimezoneLabel(timezone), + })); + } + return cachedOptions; +} + +export function buildTimezoneOptions(currentValue?: string): TimezoneOption[] { + const options = getTimezoneOptions(); + if (currentValue && !options.some((option) => option.value === currentValue)) { + return sortTimezones([currentValue, ...options.map((option) => option.value)]).map((timezone) => ({ + value: timezone, + label: formatTimezoneLabel(timezone), + })); + } + return options; +} diff --git a/grafana/dashboards/mysql/Glpi.json b/grafana/dashboards/mysql/Glpi.json new file mode 100644 index 00000000000..488a46cd23c --- /dev/null +++ b/grafana/dashboards/mysql/Glpi.json @@ -0,0 +1,1241 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "GLPI", + "url": "https://glpi-project.org/documentation/" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows basic ITSM ticket metrics from GLPI.\n- Data Source Required: GLPI", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": "mysql", + "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"GLPI entity\" (\"Choose Entity\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n case\r\n when avg_m is null then null\r\n when avg_m < 60 then concat(round(avg_m), ' minute', if(round(avg_m) = 1, '', 's'))\r\n when avg_m < 1440 then concat(round(avg_m / 60, 1), ' hour', if(round(avg_m / 60, 1) = 1, '', 's'))\r\n when avg_m < 10080 then concat(round(avg_m / 1440, 1), ' day', if(round(avg_m / 1440, 1) = 1, '', 's'))\r\n when avg_m < 43200 then concat(round(avg_m / 10080, 1), ' week', if(round(avg_m / 10080, 1) = 1, '', 's'))\r\n else concat(round(avg_m / 43200, 1), ' month', if(round(avg_m / 43200, 1) = 1, '', 's'))\r\n end as value\r\nfrom (\r\n select avg(i.lead_time_minutes) as avg_m\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n) _stats", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n),\r\n_p80 as (\r\n select max(lead_time_minutes) as p80_m\r\n from _ranks\r\n where ranks <= 0.8\r\n)\r\nselect\r\n case\r\n when p80_m is null then null\r\n when p80_m < 60 then concat(round(p80_m), ' minute', if(round(p80_m) = 1, '', 's'))\r\n when p80_m < 1440 then concat(round(p80_m / 60, 1), ' hour', if(round(p80_m / 60, 1) = 1, '', 's'))\r\n when p80_m < 10080 then concat(round(p80_m / 1440, 1), ' day', if(round(p80_m / 1440, 1) = 1, '', 's'))\r\n when p80_m < 43200 then concat(round(p80_m / 10080, 1), ' week', if(round(p80_m / 10080, 1) = 1, '', 's'))\r\n else concat(round(p80_m / 43200, 1), ' month', if(round(p80_m / 43200, 1) = 1, '', 's'))\r\n end as value\r\nfrom _p80", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Y-axis unit scales automatically (minutes, hours, days, weeks, or months) based on the largest monthly average.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(i.lead_time_minutes) as avg_m\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n),\r\n_scale as (\r\n select max(avg_m) as max_m from _requirements\r\n)\r\nselect \r\n date_format(r.time,'%M %Y') as month,\r\n case\r\n when s.max_m is null then null\r\n when s.max_m < 60 then round(r.avg_m, 1)\r\n when s.max_m < 1440 then round(r.avg_m / 60, 1)\r\n when s.max_m < 10080 then round(r.avg_m / 1440, 1)\r\n when s.max_m < 43200 then round(r.avg_m / 10080, 1)\r\n else round(r.avg_m / 43200, 1)\r\n end as mean_lead_time\r\nfrom _requirements r\r\n cross join _scale s\r\norder by r.time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "datasource": "mysql", + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 15, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.51, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\nselect \r\n now() as time,\r\n lpad(\r\n case\r\n when lead_time_minutes < 60 then concat(round(lead_time_minutes), 'm')\r\n when lead_time_minutes < 1440 then concat(round(lead_time_minutes / 60, 1), 'h')\r\n when lead_time_minutes < 10080 then concat(round(lead_time_minutes / 1440, 1), 'd')\r\n when lead_time_minutes < 43200 then concat(round(lead_time_minutes / 10080, 1), 'w')\r\n else concat(round(lead_time_minutes / 43200, 1), 'mo')\r\n end,\r\n 6, ' '\r\n ) as metric,\r\n percent_rank() over (order by lead_time_minutes asc) as value\r\nfrom _ranks\r\norder by lead_time_minutes asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "current" + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "mysql", + "definition": "select concat(name, '--', id) from boards where id like 'glpi%'", + "hide": 0, + "includeAll": true, + "label": "Choose Entity", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'glpi%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "mysql", + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "GLPI", + "uid": "glpi-dashboard", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Glpi.json b/grafana/dashboards/postgresql/Glpi.json new file mode 100644 index 00000000000..1229b4eb282 --- /dev/null +++ b/grafana/dashboards/postgresql/Glpi.json @@ -0,0 +1,1301 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "GLPI", + "url": "https://glpi-project.org/documentation/" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows basic ITSM ticket metrics from GLPI.\n- Data Source Required: GLPI", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"GLPI entity\" (\"Choose Entity\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Issues\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Delivered Issues\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN avg_m IS NULL THEN NULL WHEN avg_m < 60 THEN ROUND(avg_m) || ' minute' || CASE WHEN ROUND(avg_m) = 1 THEN '' ELSE 's' END WHEN avg_m < 1440 THEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || ' hour' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN avg_m < 10080 THEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) || ' day' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN avg_m < 43200 THEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) || ' week' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END ELSE ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) || ' month' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END END AS value FROM (SELECT AVG(i.lead_time_minutes) AS avg_m FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) AS _stats", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))), _p80 AS (SELECT MAX(lead_time_minutes) AS p80_m FROM _ranks WHERE ranks <= 0.8) SELECT CASE WHEN p80_m IS NULL THEN NULL WHEN p80_m < 60 THEN ROUND(p80_m) || ' minute' || CASE WHEN ROUND(p80_m) = 1 THEN '' ELSE 's' END WHEN p80_m < 1440 THEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || ' hour' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN p80_m < 10080 THEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) || ' day' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN p80_m < 43200 THEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) || ' week' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END ELSE ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) || ' month' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END END AS value FROM _p80", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Y-axis unit scales automatically (minutes, hours, days, weeks, or months) based on the largest monthly average.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(i.lead_time_minutes) AS avg_m FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1), _scale AS (SELECT MAX(avg_m) AS max_m FROM _requirements) SELECT TO_CHAR(CAST(r.time AS TIMESTAMP), 'FMMonth YYYY') AS month, CASE WHEN s.max_m IS NULL THEN NULL WHEN s.max_m < 60 THEN ROUND(r.avg_m, 1) WHEN s.max_m < 1440 THEN ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) WHEN s.max_m < 10080 THEN ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) WHEN s.max_m < 43200 THEN ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) ELSE ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) END AS mean_lead_time FROM _requirements AS r CROSS JOIN _scale AS s ORDER BY r.time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 15, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.51, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, LPAD(CASE WHEN lead_time_minutes < 60 THEN ROUND(lead_time_minutes) || 'm' WHEN lead_time_minutes < 1440 THEN ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || 'h' WHEN lead_time_minutes < 10080 THEN ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) || 'd' WHEN lead_time_minutes < 43200 THEN ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) || 'w' ELSE ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) || 'mo' END, 6, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_minutes ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "current" + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'glpi%'", + "hide": 0, + "includeAll": true, + "label": "Choose Entity", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'glpi%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "GLPI", + "uid": "glpi-dashboard-pg", + "version": 2, + "weekStart": "" +}