diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6df176f2740..ed05f717607 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -120,6 +120,7 @@ updates: - /backend/python - /backend/python/pydevlake - /backend/python/plugins/azuredevops + - /backend/python/plugins/redmine - /backend/python/test/fakeplugin - /grafana/scripts schedule: diff --git a/backend/python/plugins/redmine/README.md b/backend/python/plugins/redmine/README.md new file mode 100644 index 00000000000..8eaaf6e3ca7 --- /dev/null +++ b/backend/python/plugins/redmine/README.md @@ -0,0 +1,65 @@ + +# Redmine + +Apache DevLake plugin for ingesting data from [Redmine](https://www.redmine.org/) through the [REST API](https://www.redmine.org/projects/redmine/wiki/Rest_api). Each DevLake scope is a Redmine project. + +Collected data is normalized into DevLake's ticket domain so it can be used in Grafana dashboards and cross-domain metrics. + +## Collected data + +| Redmine resource | Raw table | Tool table | Domain tables | +| --- | --- | --- | --- | +| Issues | `_raw_redmine_issues` | `_tool_redmine_issues` | `issues`, `board_issues`, `issue_assignees`, `issue_labels`, `issue_relationships` | +| Journals (optional) | `_raw_redmine_issuejournals` | `_tool_redmine_journals` | `issue_comments`, `issue_changelogs` | +| Time entries (optional) | `_raw_redmine_timeentries` | `_tool_redmine_timeentries` | `issue_worklogs` | +| Projects (scope) | — | `_tool_redmine_projects` | `boards` | + +Custom field values stay on `_tool_redmine_issues.custom_fields_json`. + +Issues are collected incrementally by `updated_on`. A full sync applies the project sync policy `timeAfter` as a Redmine `created_on` filter. + +## Configuration + +1. Enable the REST API on the Redmine instance. +2. Create an API key under **My account → API access key** for a user who can see the projects to sync. +3. In Config UI, add a Redmine connection with the instance base URL and that API key. +4. Select projects and, if needed, map tracker and status names. Journals and time entries are off unless enabled on the scope config. Time entries require the Redmine permission **View time entries**. + +### Default mappings + +When a scope config does not override a name, the plugin uses these defaults. Any other tracker becomes `TASK`. Any other status becomes `OTHER`. + +| Redmine tracker | DevLake type | +| --- | --- | +| Bug | INCIDENT | +| Feature | REQUIREMENT | +| Support | TASK | +| Task | TASK | + +| Redmine status | DevLake status | +| --- | --- | +| New | TODO | +| In Progress | IN_PROGRESS | +| Feedback | IN_PROGRESS | +| Resolved | DONE | +| Closed | DONE | +| Rejected | DONE | + +## Dashboard + +After a blueprint run, open the Redmine dashboard in Grafana. MySQL and PostgreSQL definitions are in `grafana/dashboards/mysql/Redmine.json` and `grafana/dashboards/postgresql/Redmine.json`. diff --git a/backend/python/plugins/redmine/build.sh b/backend/python/plugins/redmine/build.sh new file mode 100755 index 00000000000..43613c9878a --- /dev/null +++ b/backend/python/plugins/redmine/build.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "$(dirname "$0")" +sh ../../uv.sh sync . diff --git a/backend/python/plugins/redmine/pyproject.toml b/backend/python/plugins/redmine/pyproject.toml new file mode 100644 index 00000000000..aa1b847ecb0 --- /dev/null +++ b/backend/python/plugins/redmine/pyproject.toml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[tool.poetry] +name = "redmine" +version = "0.1.0" +description = "DevLake plugin for Redmine" +authors = ["Roger Abelenda <3179183+rabelenda@users.noreply.github.com>"] +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.11,<3.12" +pydevlake = { path = "../../pydevlake", develop = true } + +[tool.poetry.group.dev.dependencies] +pytest = "^9.1.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/backend/python/plugins/redmine/redmine/__init__.py b/backend/python/plugins/redmine/redmine/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/plugins/redmine/redmine/api.py b/backend/python/plugins/redmine/redmine/api.py new file mode 100644 index 00000000000..bc80ee1c317 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/api.py @@ -0,0 +1,181 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterator, Optional, Union + +from pydevlake.api import API, Paginator, PagedResponse, Request, request_hook + + +_PAGE_SIZE = 100 +_ISSUE_SORT_UPDATED_ON = 'updated_on:desc' + + +class RedminePaginator(Paginator): + def get_items(self, response) -> list: + for value in response.json.values(): + if isinstance(value, list): + return value + return [] + + def get_next_page_id(self, response) -> Optional[int]: + data = response.json + items = self.get_items(response) + total = data.get('total_count', 0) + offset = data.get('offset', 0) + next_offset = offset + len(items) + return next_offset if next_offset < total else None + + def set_next_page_param(self, request, next_page_id): + request.query_args['offset'] = next_page_id + + +class RedmineAPI(API): + paginator = RedminePaginator() + + @property + def base_url(self) -> str: + return str(self.connection.endpoint).rstrip('/') + + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + @request_hook + def authenticate(self, request: Request): + request.headers['X-Redmine-API-Key'] = self.connection.token.get_secret_value() + + def test_connection(self): + self.get('projects.json', limit=1) + + def projects(self): + return self.get('projects.json', limit=_PAGE_SIZE) + + def issues( + self, + project_id: str, + include_journals: bool = False, + include_relations: bool = False, + created_on: Optional[str] = None, + updated_since: Optional[str] = None, + limit: Optional[int] = None, + ) -> Iterator[dict]: + yield from self.iter_issues( + project_id, + include_journals=include_journals, + include_relations=include_relations, + created_on=created_on, + updated_since=updated_since, + limit=limit, + ) + + def iter_issues( + self, + project_id: str, + include_journals: bool = False, + include_relations: bool = False, + created_on: Optional[str] = None, + updated_since: Optional[str] = None, + limit: Optional[int] = None, + ) -> Iterator[dict]: + params = self._issue_params( + project_id, + include_journals=include_journals, + include_relations=include_relations, + created_on=created_on, + ) + if updated_since: + yield from self._iter_issues( + params, + limit=limit, + stop_before_updated_on=updated_since, + sort=_ISSUE_SORT_UPDATED_ON, + ) + return + if limit is not None: + yield from self._iter_issues(params, limit=limit) + return + yield from self.get('issues.json', **params) + + def issue(self, issue_id: Union[int, str], include_journals: bool = False) -> dict: + params = {} + if include_journals: + params['include'] = 'journals' + response = self.get(f'issues/{issue_id}.json', **params) + return response.json['issue'] + + @staticmethod + def _issue_params( + project_id: str, + include_journals: bool = False, + include_relations: bool = False, + created_on: Optional[str] = None, + ) -> dict: + params = {'project_id': project_id, 'status_id': '*', 'limit': _PAGE_SIZE} + includes = [] + if include_journals: + includes.append('journals') + if include_relations: + includes.append('relations') + if includes: + params['include'] = ','.join(includes) + if created_on: + params['created_on'] = created_on + return params + + @staticmethod + def _should_stop_issue(issue: dict, stop_before_updated_on: Optional[str]) -> bool: + if stop_before_updated_on: + updated_on = issue.get('updated_on') + if updated_on and updated_on <= stop_before_updated_on: + return True + return False + + def _iter_issues( + self, + params: dict, + limit: Optional[int] = None, + stop_before_updated_on: Optional[str] = None, + sort: Optional[str] = None, + ) -> Iterator[dict]: + query = dict(params) + if sort: + query['sort'] = sort + yielded = 0 + offset = 0 + while limit is None or yielded < limit: + query['limit'] = _PAGE_SIZE if limit is None else min(_PAGE_SIZE, limit - yielded) + if offset: + query['offset'] = offset + elif 'offset' in query: + del query['offset'] + paged = self.get('issues.json', **query) + items = paged.items or [] + if not items: + break + for item in items: + if self._should_stop_issue(item, stop_before_updated_on): + return + yield item + yielded += 1 + if limit is not None and yielded >= limit: + return + if paged.next_page_request is None: + break + offset += len(items) + + def time_entries(self, project_id: str): + return self.get('time_entries.json', project_id=project_id, limit=_PAGE_SIZE) diff --git a/backend/python/plugins/redmine/redmine/main.py b/backend/python/plugins/redmine/redmine/main.py new file mode 100644 index 00000000000..861c1b36f57 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/main.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from redmine.api import RedmineAPI +from redmine.models import Connection, Project, ScopeConfig +from redmine.streams.issues import Issues +from redmine.streams.issue_journals import IssueJournals +from redmine.streams.time_entries import TimeEntries + +from pydevlake import Plugin, RemoteScopeGroup, TestConnectionResult +from pydevlake.logger import logger +import pydevlake.domain_layer.ticket as ticket + +ROOT_GROUP_ID = 'root' + + +class RedminePlugin(Plugin): + + @property + def connection_type(self): + return Connection + + @property + def tool_scope_type(self): + return Project + + @property + def scope_config_type(self): + return ScopeConfig + + def domain_scopes(self, project: Project): + yield ticket.Board( + name=project.name, + type='Scrum', + ) + + def remote_scope_groups(self, connection) -> list[RemoteScopeGroup]: + return [RemoteScopeGroup(id=ROOT_GROUP_ID, name='Projects')] + + def remote_scopes(self, connection, group_id: str) -> list[Project]: + with RedmineAPI(connection) as api: + for raw in api.projects(): + yield Project( + id=str(raw['id']), + name=raw.get('name', ''), + identifier=raw.get('identifier'), + description=raw.get('description'), + ) + + def test_connection(self, connection: Connection) -> TestConnectionResult: + with RedmineAPI(connection) as api: + try: + api.test_connection() + return TestConnectionResult(success=True, message='Connection successful', status=200) + except Exception as e: + logger.warning("Redmine test_connection failed", exc_info=True) + return TestConnectionResult(success=False, message=str(e), status=400) + + @property + def streams(self): + return [Issues, IssueJournals, TimeEntries] + + +if __name__ == '__main__': + RedminePlugin.start() diff --git a/backend/python/plugins/redmine/redmine/migrations.py b/backend/python/plugins/redmine/redmine/migrations.py new file mode 100644 index 00000000000..604cb870330 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/migrations.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from typing import Dict, Optional + +from pydantic import AnyUrl, SecretStr + +from pydevlake import Field +from pydevlake.migration import migration, MigrationScriptBuilder +from pydevlake.model import Connection as BaseConnection, Model, ToolModel, ToolScope, ToolTable + + +@migration(20260609000001, name="initialize schemas for Redmine") +def init_schemas(b: MigrationScriptBuilder): + class Connection(BaseConnection): + endpoint: AnyUrl + token: SecretStr + + class Project(ToolScope): + identifier: Optional[str] + description: Optional[str] + + class ScopeConfig(ToolTable, Model): + name: str = Field(default="default") + entities: Optional[str] + connection_id: Optional[int] + collect_journals: bool = False + collect_time_entries: bool = False + tickets_created_after: Optional[datetime.datetime] = None + type_mappings: Optional[Dict] = None + status_mappings: Optional[Dict] = None + + class Issue(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + project_id: Optional[int] + tracker_name: Optional[str] + status_name: Optional[str] + priority_name: Optional[str] + author_id: Optional[int] + author_name: Optional[str] + assigned_to_id: Optional[int] + assigned_to_name: Optional[str] + parent_id: Optional[int] + category_id: Optional[int] + category_name: Optional[str] + fixed_version_id: Optional[int] + fixed_version_name: Optional[str] + subject: Optional[str] + description: Optional[str] + start_date: Optional[datetime.date] + due_date: Optional[datetime.date] + done_ratio: Optional[int] + estimated_hours: Optional[float] + spent_hours: Optional[float] + total_spent_hours: Optional[float] + custom_fields_json: Optional[str] + relations_json: Optional[str] + created_on: Optional[datetime.datetime] + updated_on: Optional[datetime.datetime] + closed_on: Optional[datetime.datetime] + + class Journal(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + issue_id: Optional[int] + user_id: Optional[int] + user_name: Optional[str] + notes: Optional[str] + details_json: Optional[str] + created_on: Optional[datetime.datetime] + private_notes: Optional[bool] + + class TimeEntry(ToolModel): + id: int = Field(primary_key=True, auto_increment=False) + project_id: Optional[int] + issue_id: Optional[int] + user_id: Optional[int] + user_name: Optional[str] + hours: Optional[float] + comments: Optional[str] + spent_on: Optional[datetime.date] + created_on: Optional[datetime.datetime] + updated_on: Optional[datetime.datetime] + + b.create_tables( + Connection, + Project, + ScopeConfig, + Issue, + Journal, + TimeEntry, + ) diff --git a/backend/python/plugins/redmine/redmine/models.py b/backend/python/plugins/redmine/redmine/models.py new file mode 100644 index 00000000000..2c81a4b3f0d --- /dev/null +++ b/backend/python/plugins/redmine/redmine/models.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from typing import Dict, Optional + +from pydantic import AnyUrl, BaseModel, SecretStr + +from pydevlake import Field, ScopeConfig as BaseScopeConfig +from pydevlake.model import Connection as BaseConnection, ToolModel, ToolScope + +from redmine.migrations import * + + +class StatusMapping(BaseModel): + standard_status: str = Field(alias='standardStatus') + + class Config: + allow_population_by_field_name = True + + +class TypeMapping(BaseModel): + standard_type: str = Field(alias='standardType') + + class Config: + allow_population_by_field_name = True + + +class Connection(BaseConnection): + endpoint: AnyUrl + token: SecretStr + + +class ScopeConfig(BaseScopeConfig): + collect_journals: bool = False + collect_time_entries: bool = False + type_mappings: Dict[str, TypeMapping] = Field(default_factory=dict, alias='typeMappings') + status_mappings: Dict[str, StatusMapping] = Field(default_factory=dict, alias='statusMappings') + + +class Project(ToolScope, table=True): + identifier: Optional[str] + description: Optional[str] + + +class Issue(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + project_id: Optional[int] + tracker_name: Optional[str] + status_name: Optional[str] + priority_name: Optional[str] + author_id: Optional[int] + author_name: Optional[str] + assigned_to_id: Optional[int] + assigned_to_name: Optional[str] + parent_id: Optional[int] + category_id: Optional[int] + category_name: Optional[str] + fixed_version_id: Optional[int] + fixed_version_name: Optional[str] + subject: Optional[str] + description: Optional[str] + start_date: Optional[datetime.date] + due_date: Optional[datetime.date] + done_ratio: Optional[int] + estimated_hours: Optional[float] + spent_hours: Optional[float] + total_spent_hours: Optional[float] + custom_fields_json: Optional[str] + relations_json: Optional[str] + created_on: Optional[datetime.datetime] + updated_on: Optional[datetime.datetime] + closed_on: Optional[datetime.datetime] + + +class Journal(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + issue_id: Optional[int] + user_id: Optional[int] + user_name: Optional[str] + notes: Optional[str] + details_json: Optional[str] + created_on: Optional[datetime.datetime] + private_notes: Optional[bool] + + +class TimeEntry(ToolModel, table=True): + id: int = Field(primary_key=True, auto_increment=False) + project_id: Optional[int] + issue_id: Optional[int] + user_id: Optional[int] + user_name: Optional[str] + hours: Optional[float] + comments: Optional[str] + spent_on: Optional[datetime.date] + created_on: Optional[datetime.datetime] + updated_on: Optional[datetime.datetime] diff --git a/backend/python/plugins/redmine/redmine/streams/__init__.py b/backend/python/plugins/redmine/redmine/streams/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/streams/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/plugins/redmine/redmine/streams/issue_journals.py b/backend/python/plugins/redmine/redmine/streams/issue_journals.py new file mode 100644 index 00000000000..14a34ba6c54 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/streams/issue_journals.py @@ -0,0 +1,160 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +from typing import Iterable, Optional + +from redmine.api import RedmineAPI +from redmine.models import Issue, Journal, Project +from redmine.streams.issues import Issues +import pydevlake.domain_layer.ticket as ticket +from pydevlake import Stream, DomainType, domain_id +from pydevlake.collection_time import collection_time_after, format_collection_date + + +_ATTR_FIELD_MAP = { + 'assigned_to_id': 'assignee', + 'status_id': 'status', + 'priority_id': 'priority', + 'category_id': 'category', + 'fixed_version_id': 'fix_version', + 'tracker_id': 'type', + 'parent_issue_id': 'parent', + 'done_ratio': 'progress', + 'estimated_hours': 'original_estimate', + 'start_date': 'start_date', + 'due_date': 'due_date', + 'subject': 'title', + 'description': 'description', + 'is_private': 'is_private', +} + + +class IssueJournals(Stream): + tool_model = Journal + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueComment, ticket.IssueChangelogs] + + def collect(self, state, context) -> Iterable[tuple[object, dict]]: + if not getattr(context.scope_config, 'collect_journals', False): + return + project: Project = context.scope + run_state = dict(state) + use_incremental = context.options.get('incremental') + updated_since = run_state.get('updated_since') if use_incremental else None + created_on = self._created_on_filter(context) + with RedmineAPI(context.connection) as api: + for raw_issue in api.issues( + project.id, + created_on=created_on, + updated_since=updated_since, + ): + Issues._track_updated_since(run_state, raw_issue.get('updated_on')) + issue_id = raw_issue['id'] + full_issue = api.issue(issue_id, include_journals=True) + for journal in full_issue.get('journals', []): + journal['_issue_id'] = issue_id + journal['x_request_url'] = f'{api.base_url}/issues/{issue_id}.json' + yield journal, run_state + + @staticmethod + def _created_on_filter(context) -> Optional[str]: + if context.options.get('incremental'): + return None + time_after = collection_time_after(context) + if not time_after: + return None + return f'>={format_collection_date(time_after)}' + + def extract(self, raw: dict) -> Journal: + details = raw.get('details') + return Journal( + id=raw['id'], + issue_id=raw.get('_issue_id'), + user_id=raw.get('user', {}).get('id') if raw.get('user') else None, + user_name=raw.get('user', {}).get('name') if raw.get('user') else None, + notes=raw.get('notes'), + details_json=json.dumps(details) if details else None, + created_on=raw.get('created_on'), + private_notes=raw.get('private_notes', False), + ) + + def convert(self, j: Journal, ctx) -> Iterable[object]: + if not j.issue_id: + return + issue_id = domain_id(Issue, ctx.connection.id, j.issue_id) + if j.notes: + yield ticket.IssueComment( + issue_id=issue_id, + body=j.notes, + account_id=str(j.user_id) if j.user_id else None, + created_date=j.created_on, + ) + for idx, detail in enumerate(self._details(j.details_json)): + changelog = self._detail_to_changelog(j, issue_id, detail, idx, ctx) + if changelog is not None: + yield changelog + + @staticmethod + def _details(details_json: Optional[str]) -> list[dict]: + if not details_json: + return [] + try: + details = json.loads(details_json) + except json.JSONDecodeError: + return [] + return details if isinstance(details, list) else [] + + def _detail_to_changelog( + self, + journal: Journal, + issue_id: str, + detail: dict, + idx: int, + ctx, + ) -> Optional[ticket.IssueChangelogs]: + field_name = self._changelog_field_name(detail.get('property'), detail.get('name')) + if not field_name: + return None + old_value = detail.get('old_value') + new_value = detail.get('new_value') + if old_value in (None, '') and new_value in (None, ''): + return None + return ticket.IssueChangelogs( + id=domain_id(Journal, ctx.connection.id, journal.id, idx), + issue_id=issue_id, + author_id=str(journal.user_id) if journal.user_id else None, + author_name=journal.user_name, + field_id=detail.get('name'), + field_name=field_name, + original_from_value=old_value, + original_to_value=new_value, + from_value=old_value, + to_value=new_value, + created_date=journal.created_on, + ) + + @staticmethod + def _changelog_field_name(property_name: Optional[str], name: Optional[str]) -> Optional[str]: + if not name: + return None + if property_name == 'attr': + return _ATTR_FIELD_MAP.get(name, name) + if property_name == 'cf': + return f'custom_field:{name}' + if property_name == 'relation': + return 'relation' + return name diff --git a/backend/python/plugins/redmine/redmine/streams/issues.py b/backend/python/plugins/redmine/redmine/streams/issues.py new file mode 100644 index 00000000000..31dc224f534 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/streams/issues.py @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import json +from typing import Iterable, Optional + +from redmine.api import RedmineAPI +from redmine.models import Issue, Project, ScopeConfig, StatusMapping, TypeMapping +import pydevlake.domain_layer.ticket as ticket +from pydevlake import Stream, DomainType, domain_id +from pydevlake.collection_time import collection_time_after, format_collection_date + +_DEFAULT_TYPE_BY_TRACKER = { + 'Bug': 'INCIDENT', + 'Feature': 'REQUIREMENT', + 'Support': 'TASK', + 'Task': 'TASK', +} + +_DEFAULT_STATUS_BY_NAME = { + 'New': 'TODO', + 'In Progress': 'IN_PROGRESS', + 'Feedback': 'IN_PROGRESS', + 'Resolved': 'DONE', + 'Closed': 'DONE', + 'Rejected': 'DONE', +} + + +class Issues(Stream): + tool_model = Issue + domain_types = [DomainType.TICKET] + domain_models = [ + ticket.Issue, + ticket.BoardIssue, + ticket.IssueAssignee, + ticket.IssueLabel, + ticket.IssueRelationship, + ] + + def collect(self, state, context) -> Iterable[tuple[object, dict]]: + project: Project = context.scope + run_state = dict(state) + use_incremental = context.options.get('incremental') + updated_since = run_state.get('updated_since') if use_incremental else None + created_on = None + if not use_incremental: + time_after = collection_time_after(context) + if time_after: + created_on = f'>={format_collection_date(time_after)}' + with RedmineAPI(context.connection) as api: + for raw in api.issues( + project.id, + created_on=created_on, + updated_since=updated_since, + include_relations=True, + ): + self._track_updated_since(run_state, raw.get('updated_on')) + raw['x_request_url'] = f'{api.base_url}/issues.json' + raw['x_request_input'] = {'project_id': project.id} + if created_on: + raw['x_request_input']['created_on'] = created_on + if updated_since: + raw['x_request_input']['updated_since'] = updated_since + yield raw, run_state + + @staticmethod + def _track_updated_since(state: dict, updated_on: Optional[str]) -> None: + if not updated_on: + return + current = state.get('updated_since') + if current is None or updated_on > current: + state['updated_since'] = updated_on + + def extract(self, raw: dict) -> Issue: + parent = raw.get('parent') or {} + category = raw.get('category') or {} + fixed_version = raw.get('fixed_version') or {} + custom_fields = raw.get('custom_fields') + relations = raw.get('relations') + return Issue( + id=raw['id'], + project_id=raw.get('project', {}).get('id'), + tracker_name=raw.get('tracker', {}).get('name'), + status_name=raw.get('status', {}).get('name'), + priority_name=raw.get('priority', {}).get('name'), + author_id=raw.get('author', {}).get('id'), + author_name=raw.get('author', {}).get('name'), + assigned_to_id=raw.get('assigned_to', {}).get('id') if raw.get('assigned_to') else None, + assigned_to_name=raw.get('assigned_to', {}).get('name') if raw.get('assigned_to') else None, + parent_id=parent.get('id'), + category_id=category.get('id'), + category_name=category.get('name'), + fixed_version_id=fixed_version.get('id'), + fixed_version_name=fixed_version.get('name'), + subject=raw.get('subject'), + description=raw.get('description'), + start_date=raw.get('start_date'), + due_date=raw.get('due_date'), + done_ratio=raw.get('done_ratio'), + estimated_hours=raw.get('estimated_hours'), + spent_hours=raw.get('spent_hours'), + total_spent_hours=raw.get('total_spent_hours'), + custom_fields_json=json.dumps(custom_fields) if custom_fields else None, + relations_json=json.dumps(relations) if relations else None, + created_on=raw.get('created_on'), + updated_on=raw.get('updated_on'), + closed_on=raw.get('closed_on'), + ) + + def convert(self, issue: Issue, ctx) -> Iterable[object]: + board_id = ctx.scope.domain_id() + tracker = issue.tracker_name or 'Task' + issue_id = domain_id(Issue, ctx.connection.id, issue.id) + original_estimate_minutes = self._hours_to_minutes(issue.estimated_hours) + time_spent_minutes = self._hours_to_minutes(issue.spent_hours) + time_remaining_minutes = None + if original_estimate_minutes is not None and time_spent_minutes is not None: + time_remaining_minutes = max(0, original_estimate_minutes - time_spent_minutes) + i = ticket.Issue( + url=self._issue_url(ctx.connection.endpoint, issue.id), + issue_key=str(issue.id), + title=issue.subject or '', + description=issue.description, + type=self._resolve_issue_type(issue.tracker_name, ctx.scope_config), + original_type=tracker, + status=self._resolve_issue_status(issue.status_name, ctx.scope_config), + original_status=issue.status_name, + created_date=issue.created_on, + updated_date=issue.updated_on, + resolution_date=issue.closed_on, + due_date=self._due_date(issue.due_date), + creator_id=str(issue.author_id) if issue.author_id else None, + creator_name=issue.author_name, + assignee_id=str(issue.assigned_to_id) if issue.assigned_to_id else None, + assignee_name=issue.assigned_to_name, + parent_issue_id=( + domain_id(Issue, ctx.connection.id, issue.parent_id) + if issue.parent_id else None + ), + is_subtask=issue.parent_id is not None, + priority=issue.priority_name, + component=issue.category_name, + original_project=ctx.scope.name, + lead_time_minutes=self._lead_time_minutes(issue), + original_estimate_minutes=original_estimate_minutes, + time_spent_minutes=time_spent_minutes, + time_remaining_minutes=time_remaining_minutes, + fix_versions=issue.fixed_version_name, + ) + yield i + yield ticket.BoardIssue(board_id=board_id, issue_id=i.id) + if issue.assigned_to_id: + yield ticket.IssueAssignee( + issue_id=i.id, + assignee_id=str(issue.assigned_to_id), + assignee_name=issue.assigned_to_name, + ) + if issue.category_name: + yield ticket.IssueLabel(issue_id=issue_id, label_name=issue.category_name) + for relation in self._relations(issue.relations_json): + source_id = relation.get('issue_id') + target_id = relation.get('issue_to_id') + relation_type = relation.get('relation_type') + if not (source_id and target_id and relation_type): + continue + yield ticket.IssueRelationship( + source_issue_id=domain_id(Issue, ctx.connection.id, source_id), + target_issue_id=domain_id(Issue, ctx.connection.id, target_id), + original_type=relation_type, + ) + + @staticmethod + def _issue_url(endpoint, issue_id: int) -> str: + return f'{str(endpoint).rstrip("/")}/issues/{issue_id}' + + @staticmethod + def _hours_to_minutes(hours: Optional[float]) -> Optional[int]: + if hours is None: + return None + return round(hours * 60) + + def _resolve_issue_type(self, tracker_name: Optional[str], scope_config: ScopeConfig) -> str: + tracker = tracker_name or 'Task' + configured = self._standard_type((scope_config.type_mappings or {}).get(tracker)) + if configured: + return configured.upper() + return _DEFAULT_TYPE_BY_TRACKER.get(tracker, 'TASK') + + def _standard_type(self, entry) -> Optional[str]: + if entry is None: + return None + if isinstance(entry, TypeMapping): + return entry.standard_type + if isinstance(entry, dict): + value = entry.get('standardType') or entry.get('standard_type') + return str(value) if value else None + return None + + def _resolve_issue_status(self, status_name: Optional[str], scope_config: ScopeConfig) -> str: + if not status_name: + return 'OTHER' + configured = self._standard_status((scope_config.status_mappings or {}).get(status_name)) + if configured: + return configured.upper() + return _DEFAULT_STATUS_BY_NAME.get(status_name, 'OTHER') + + def _standard_status(self, entry) -> Optional[str]: + if entry is None: + return None + if isinstance(entry, StatusMapping): + return entry.standard_status + if isinstance(entry, dict): + value = entry.get('standardStatus') or entry.get('standard_status') + return str(value) if value else None + return None + + @staticmethod + def _due_date(value) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, datetime.datetime): + return value + if isinstance(value, datetime.date): + return datetime.datetime.combine(value, datetime.time.min) + if isinstance(value, str) and value: + return datetime.datetime.combine( + datetime.date.fromisoformat(value), datetime.time.min + ) + return None + + @staticmethod + def _relations(relations_json: Optional[str]) -> list[dict]: + if not relations_json: + return [] + try: + relations = json.loads(relations_json) + except json.JSONDecodeError: + return [] + return relations if isinstance(relations, list) else [] + + @staticmethod + def _parse_datetime(value) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, datetime.datetime): + return value + if isinstance(value, str): + return datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + return None + + @staticmethod + def _lead_time_minutes(issue: Issue) -> Optional[int]: + created = Issues._parse_datetime(issue.created_on) + closed = Issues._parse_datetime(issue.closed_on) + if not created or not closed: + return None + return int((closed - created).total_seconds() // 60) diff --git a/backend/python/plugins/redmine/redmine/streams/time_entries.py b/backend/python/plugins/redmine/redmine/streams/time_entries.py new file mode 100644 index 00000000000..acce8854612 --- /dev/null +++ b/backend/python/plugins/redmine/redmine/streams/time_entries.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable + +from redmine.api import RedmineAPI +from redmine.models import Issue, TimeEntry +import pydevlake.domain_layer.ticket as ticket +from pydevlake import Stream, DomainType, domain_id + + +class TimeEntries(Stream): + tool_model = TimeEntry + domain_types = [DomainType.TICKET] + domain_models = [ticket.IssueWorklog] + + def collect(self, state, context) -> Iterable[tuple[object, dict]]: + if not getattr(context.scope_config, 'collect_time_entries', False): + return + with RedmineAPI(context.connection) as api: + for raw in api.time_entries(context.scope.id): + raw['x_request_url'] = f'{api.base_url}/time_entries.json' + yield raw, state + + def extract(self, raw: dict) -> TimeEntry: + return TimeEntry( + id=raw['id'], + project_id=raw.get('project', {}).get('id') if raw.get('project') else None, + issue_id=raw.get('issue', {}).get('id') if raw.get('issue') else None, + user_id=raw.get('user', {}).get('id') if raw.get('user') else None, + user_name=raw.get('user', {}).get('name') if raw.get('user') else None, + hours=raw.get('hours'), + comments=raw.get('comments'), + spent_on=raw.get('spent_on'), + created_on=raw.get('created_on'), + updated_on=raw.get('updated_on'), + ) + + def convert(self, te: TimeEntry, ctx) -> Iterable[object]: + if not te.issue_id: + return + issue_id = domain_id(Issue, ctx.connection.id, te.issue_id) + yield ticket.IssueWorklog( + issue_id=issue_id, + author_id=str(te.user_id) if te.user_id else None, + comment=te.comments, + time_spent_minutes=round(te.hours * 60) if te.hours else None, + started_date=te.spent_on, + logged_date=te.created_on, + ) diff --git a/backend/python/plugins/redmine/run.sh b/backend/python/plugins/redmine/run.sh new file mode 100755 index 00000000000..80884bcf8d4 --- /dev/null +++ b/backend/python/plugins/redmine/run.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "$(dirname "$0")" +sh ../../uv.sh python . redmine/main.py "$@" diff --git a/backend/python/plugins/redmine/tests/__init__.py b/backend/python/plugins/redmine/tests/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/python/plugins/redmine/tests/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/plugins/redmine/tests/api_test.py b/backend/python/plugins/redmine/tests/api_test.py new file mode 100644 index 00000000000..ade6e10a074 --- /dev/null +++ b/backend/python/plugins/redmine/tests/api_test.py @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager +from typing import Iterator +from unittest.mock import MagicMock, Mock, patch + +from redmine.api import RedmineAPI, _ISSUE_SORT_UPDATED_ON + + +def _paged_mock(items, has_next=False): + paged = MagicMock() + paged.items = items + paged.next_page_request = MagicMock() if has_next else None + return paged + + +class PatchedRedmineAPI: + _AUTO_MOCK_METHODS = frozenset({'get'}) + + def __init__(self): + object.__setattr__(self, '_api', RedmineAPI(MagicMock())) + object.__setattr__(self, '_mocks', {}) + object.__setattr__(self, '_patchers', []) + + def _install_mock(self, name: str, mock: Mock) -> Mock: + self._mocks[name] = mock + patcher = patch.object(self._api, name, mock) + patcher.start() + self._patchers.append(patcher) + return mock + + def __getattr__(self, name): + if name in self._mocks: + return self._mocks[name] + if name in self._AUTO_MOCK_METHODS: + return self._install_mock(name, MagicMock()) + return getattr(self._api, name) + + def __setattr__(self, name, value): + if name.startswith('_'): + object.__setattr__(self, name, value) + elif isinstance(value, Mock): + self._install_mock(name, value) + else: + setattr(self._api, name, value) + + def __enter__(self): + return self + + def __exit__(self, *_): + for patcher in reversed(self._patchers): + patcher.stop() + + +@contextmanager +def _patched_api() -> Iterator[PatchedRedmineAPI]: + with PatchedRedmineAPI() as api: + yield api + + +def test_iter_issues_stops_after_limit_on_single_page(): + with _patched_api() as api: + api.get.return_value = _paged_mock([{'id': i} for i in range(1, 6)]) + collected = list(api._iter_issues({'project_id': '1', 'status_id': '*'}, limit=3)) + + assert [issue['id'] for issue in collected] == [1, 2, 3] + api.get.assert_called_once() + assert api.get.call_args.kwargs['limit'] == 3 + + +def test_iter_issues_stops_after_limit_across_pages(): + with _patched_api() as api: + api.get.side_effect = [ + _paged_mock([{'id': i} for i in range(1, 101)], has_next=True), + _paged_mock([{'id': i} for i in range(101, 106)]), + ] + collected = list(api._iter_issues({'project_id': '1', 'status_id': '*'}, limit=105)) + + assert len(collected) == 105 + assert api.get.call_count == 2 + assert api.get.call_args_list[0].kwargs['limit'] == 100 + assert api.get.call_args_list[1].kwargs == { + 'project_id': '1', + 'status_id': '*', + 'limit': 5, + 'offset': 100, + } + + +def test_iter_issues_stops_at_updated_since(): + issues = [ + {'id': 1, 'updated_on': '2024-02-01T10:00:00Z'}, + {'id': 2, 'updated_on': '2024-01-01T10:00:00Z'}, + {'id': 3, 'updated_on': '2023-12-01T10:00:00Z'}, + ] + + with _patched_api() as api: + api.get.return_value = _paged_mock(issues) + collected = list(api.iter_issues( + '1', + updated_since='2024-01-15T00:00:00Z', + )) + + assert [issue['id'] for issue in collected] == [1] + api.get.assert_called_once() + assert api.get.call_args.kwargs['sort'] == _ISSUE_SORT_UPDATED_ON + + +def test_iter_issues_incremental_uses_paginated_list(): + with _patched_api() as api: + api.get.side_effect = [ + _paged_mock([{'id': 1, 'updated_on': '2024-02-01T10:00:00Z'}], has_next=True), + _paged_mock([{'id': 2, 'updated_on': '2024-01-01T10:00:00Z'}]), + ] + collected = list(api.iter_issues('1', updated_since='2024-01-02T00:00:00Z')) + + assert [issue['id'] for issue in collected] == [1] + assert api.get.call_count == 2 + + +def test_issue_fetches_single_issue_with_journals(): + with _patched_api() as api: + response = MagicMock() + response.json = { + 'issue': { + 'id': 42, + 'journals': [{'id': 1, 'notes': 'done'}], + }, + } + api.get.return_value = response + + issue = api.issue(42, include_journals=True) + + assert issue['id'] == 42 + assert len(issue['journals']) == 1 + api.get.assert_called_once_with('issues/42.json', include='journals') diff --git a/backend/python/plugins/redmine/tests/collection_test.py b/backend/python/plugins/redmine/tests/collection_test.py new file mode 100644 index 00000000000..6e0f4d6a0eb --- /dev/null +++ b/backend/python/plugins/redmine/tests/collection_test.py @@ -0,0 +1,178 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from unittest.mock import MagicMock, patch + +from pydevlake.testing import ContextBuilder + +from redmine.main import RedminePlugin +from redmine.streams.issue_journals import IssueJournals +from redmine.streams.issues import Issues +from redmine.streams.time_entries import TimeEntries + + +def _collect_context(**scope_config_kwargs): + builder = ( + ContextBuilder(RedminePlugin()) + .with_connection(endpoint='http://redmine.example.com', token='test_api_key') + .with_scope('1', name='Test Project') + .with_scope_config(**scope_config_kwargs) + ) + return builder.build() + + +@patch('redmine.streams.issues.RedmineAPI') +def test_collect_uses_project_time_after_when_not_incremental(RedmineAPI): + context = ( + ContextBuilder(RedminePlugin()) + .with_connection(endpoint='http://redmine.example.com', token='test_api_key') + .with_scope('1', name='Test Project') + .with_time_after(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc)) + .build() + ) + api = MagicMock() + api.issues.return_value = iter([]) + api.base_url = 'http://redmine.example.com' + api.__enter__.return_value = api + api.__exit__.return_value = False + RedmineAPI.return_value = api + + list(Issues(RedminePlugin().name).collect({}, context)) + + api.issues.assert_called_once_with( + '1', + created_on='>=2024-01-01', + updated_since=None, + include_relations=True, + ) + + +@patch('redmine.streams.issues.RedmineAPI') +def test_collect_without_time_after_omits_filter(RedmineAPI): + context = _collect_context() + api = MagicMock() + api.issues.return_value = iter([]) + api.base_url = 'http://redmine.example.com' + api.__enter__.return_value = api + api.__exit__.return_value = False + RedmineAPI.return_value = api + + list(Issues(RedminePlugin().name).collect({}, context)) + + api.issues.assert_called_once_with( + '1', + created_on=None, + updated_since=None, + include_relations=True, + ) + + +@patch('redmine.streams.issues.RedmineAPI') +def test_collect_uses_updated_since_when_incremental(RedmineAPI): + context = _collect_context() + context.options['incremental'] = True + api = MagicMock() + api.issues.return_value = iter([ + {'id': 1, 'updated_on': '2024-06-02T10:00:00Z'}, + ]) + api.base_url = 'http://redmine.example.com' + api.__enter__.return_value = api + api.__exit__.return_value = False + RedmineAPI.return_value = api + + _, state = next(Issues(RedminePlugin().name).collect( + {'updated_since': '2024-06-01T00:00:00Z'}, + context, + )) + + api.issues.assert_called_once_with( + '1', + created_on=None, + updated_since='2024-06-01T00:00:00Z', + include_relations=True, + ) + assert state['updated_since'] == '2024-06-02T10:00:00Z' + + +@patch('redmine.streams.time_entries.RedmineAPI') +def test_collect_time_entries_skips_when_disabled(RedmineAPI): + context = _collect_context() + api = MagicMock() + api.time_entries.return_value = iter([{'id': 1}]) + api.base_url = 'http://redmine.example.com' + api.__enter__.return_value = api + api.__exit__.return_value = False + RedmineAPI.return_value = api + + collected = list(TimeEntries(RedminePlugin().name).collect({}, context)) + + assert collected == [] + RedmineAPI.assert_not_called() + + +@patch('redmine.streams.time_entries.RedmineAPI') +def test_collect_time_entries_fetches_when_enabled(RedmineAPI): + context = _collect_context(collect_time_entries=True) + api = MagicMock() + api.time_entries.return_value = iter([{'id': 1}]) + api.base_url = 'http://redmine.example.com' + api.__enter__.return_value = api + api.__exit__.return_value = False + RedmineAPI.return_value = api + + list(TimeEntries(RedminePlugin().name).collect({}, context)) + + api.time_entries.assert_called_once_with('1') + + +@patch('redmine.streams.issue_journals.RedmineAPI') +def test_collect_issue_journals_skips_when_disabled(RedmineAPI): + context = _collect_context() + collected = list(IssueJournals(RedminePlugin().name).collect({}, context)) + + assert collected == [] + RedmineAPI.assert_not_called() + + +@patch('redmine.streams.issue_journals.RedmineAPI') +def test_collect_issue_journals_fetches_each_issue_with_journals(RedmineAPI): + context = _collect_context(collect_journals=True) + api = MagicMock() + api.issues.return_value = iter([{'id': 42}, {'id': 43}]) + api.issue.side_effect = [ + {'id': 42, 'journals': [{'id': 1, 'notes': 'a', 'details': []}]}, + {'id': 43, 'journals': []}, + ] + api.base_url = 'http://redmine.example.com' + api.__enter__.return_value = api + api.__exit__.return_value = False + RedmineAPI.return_value = api + + collected = list(IssueJournals(RedminePlugin().name).collect({}, context)) + + assert len(collected) == 1 + journal, _ = collected[0] + assert journal['id'] == 1 + assert journal['_issue_id'] == 42 + api.issues.assert_called_once_with( + '1', + created_on=None, + updated_since=None, + ) + assert api.issue.call_args_list == [ + ((42,), {'include_journals': True}), + ((43,), {'include_journals': True}), + ] diff --git a/backend/python/plugins/redmine/tests/plugin_test.py b/backend/python/plugins/redmine/tests/plugin_test.py new file mode 100644 index 00000000000..20085ee9230 --- /dev/null +++ b/backend/python/plugins/redmine/tests/plugin_test.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest + +from pydevlake.testing import assert_valid_plugin, assert_plugin_run + +from redmine.main import RedminePlugin +from redmine.models import Connection, ScopeConfig + + +def test_valid_plugin(): + assert_valid_plugin(RedminePlugin()) + + +def test_valid_plugin_and_connection(): + endpoint = os.environ.get('REDMINE_ENDPOINT') + token = os.environ.get('REDMINE_TOKEN') + if not (endpoint and token): + pytest.skip("REDMINE_ENDPOINT and REDMINE_TOKEN env vars required") + + connection = Connection(id=1, name='test_connection', endpoint=endpoint, token=token) + scope_config = ScopeConfig(id=1, name='test_config') + assert_plugin_run(RedminePlugin(), connection, scope_config) diff --git a/backend/python/plugins/redmine/tests/remote_scopes_test.py b/backend/python/plugins/redmine/tests/remote_scopes_test.py new file mode 100644 index 00000000000..4980c4a0d2d --- /dev/null +++ b/backend/python/plugins/redmine/tests/remote_scopes_test.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock, patch + +from redmine.main import ROOT_GROUP_ID, RedminePlugin +from redmine.models import Connection + +SAMPLE_PROJECTS = [ + {'id': 1, 'name': 'Alpha', 'identifier': 'alpha'}, + {'id': 2, 'name': 'Beta', 'identifier': 'beta'}, +] + + +def _connection(): + return Connection(id=1, name='test', endpoint='https://redmine.example.com', token='token') + + +@patch('redmine.main.RedmineAPI') +def test_remote_scopes_returns_all_projects(RedmineAPI): + plugin = RedminePlugin() + api = MagicMock() + api.projects.return_value = SAMPLE_PROJECTS + api.__enter__.return_value = api + api.__exit__.return_value = False + RedmineAPI.return_value = api + + projects = list(plugin.remote_scopes(_connection(), ROOT_GROUP_ID)) + + assert [project.name for project in projects] == ['Alpha', 'Beta'] + + +@patch('redmine.main.RedmineAPI') +def test_remote_scope_groups_returns_projects_group(RedmineAPI): + plugin = RedminePlugin() + groups = list(plugin.remote_scope_groups(_connection())) + + assert len(groups) == 1 + assert groups[0].id == ROOT_GROUP_ID + assert groups[0].name == 'Projects' diff --git a/backend/python/plugins/redmine/tests/streams_test.py b/backend/python/plugins/redmine/tests/streams_test.py new file mode 100644 index 00000000000..d13a07dd198 --- /dev/null +++ b/backend/python/plugins/redmine/tests/streams_test.py @@ -0,0 +1,415 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +import pytest + +import pydevlake.domain_layer.ticket as ticket +from pydevlake.model import domain_id +from pydevlake.testing import assert_stream_convert, ContextBuilder + +from redmine.main import RedminePlugin +from redmine.models import Issue, Journal, StatusMapping, TypeMapping + + +def _issue_raw(issue_id: int, tracker_name: str, status_name: str, subject: str = 'Test issue') -> dict: + return { + 'id': issue_id, + 'project': {'id': 1, 'name': 'Test Project'}, + 'tracker': {'id': 1, 'name': tracker_name}, + 'status': {'id': 1, 'name': status_name}, + 'priority': {'id': 3, 'name': 'Normal'}, + 'author': {'id': 10, 'name': 'John Smith'}, + 'subject': subject, + 'created_on': '2024-02-01T10:00:00Z', + 'updated_on': '2024-02-01T10:00:00Z', + 'x_request_url': 'http://redmine.example.com/issues.json', + 'x_request_input': {'project_id': '1'}, + } + + +def _expected_issue(context, issue_id, tracker_name, status_name, expected_type, expected_status, subject='Test issue'): + i = ticket.Issue( + url=f'http://redmine.example.com/issues/{issue_id}', + issue_key=str(issue_id), + title=subject, + description=None, + type=expected_type, + original_type=tracker_name, + status=expected_status, + original_status=status_name, + created_date='2024-02-01T10:00:00Z', + updated_date='2024-02-01T10:00:00Z', + creator_id='10', + creator_name='John Smith', + priority='Normal', + original_project='Test Project', + ) + return [i, ticket.BoardIssue(board_id=context.scope.domain_id(), issue_id=i.id)] + + +@pytest.fixture +def context(): + return ( + ContextBuilder(RedminePlugin()) + .with_connection(endpoint='http://redmine.example.com', token='test_api_key') + .with_scope('1', name='Test Project') + .with_scope_config() + .build() + ) + + +@pytest.fixture +def context_with_journals(): + return ( + ContextBuilder(RedminePlugin()) + .with_connection(endpoint='http://redmine.example.com', token='test_api_key') + .with_scope('1', name='Test Project') + .with_scope_config(collect_journals=True) + .build() + ) + + +def test_issues_stream(context): + raw = { + 'id': 42, + 'project': {'id': 1, 'name': 'Test Project'}, + 'tracker': {'id': 1, 'name': 'Bug'}, + 'status': {'id': 5, 'name': 'Resolved'}, + 'priority': {'id': 4, 'name': 'High'}, + 'author': {'id': 10, 'name': 'John Smith'}, + 'assigned_to': {'id': 11, 'name': 'Jane Doe'}, + 'parent': {'id': 40, 'name': 'Parent issue'}, + 'category': {'id': 9, 'name': 'Authentication'}, + 'fixed_version': {'id': 3, 'name': '1.0.0'}, + 'subject': 'Cannot login to the app', + 'description': 'Login fails with 500 error.', + 'start_date': '2024-01-10', + 'due_date': '2024-01-20', + 'done_ratio': 100, + 'estimated_hours': 4.0, + 'spent_hours': 2.5, + 'total_spent_hours': 3.0, + 'custom_fields': [ + {'id': 1, 'name': 'Resolution', 'value': 'Fixed'}, + {'id': 2, 'name': 'Tags', 'value': ['backend', 'security']}, + ], + 'relations': [ + { + 'id': 1819, + 'issue_id': 42, + 'issue_to_id': 43, + 'relation_type': 'blocks', + } + ], + 'created_on': '2024-01-10T09:00:00Z', + 'updated_on': '2024-01-15T14:00:00Z', + 'closed_on': '2024-01-15T14:00:00Z', + 'x_request_url': 'http://redmine.example.com/issues.json', + 'x_request_input': {'project_id': '1'}, + } + + issue_id = domain_id(Issue, context.connection.id, 42) + i = ticket.Issue( + url='http://redmine.example.com/issues/42', + issue_key='42', + title='Cannot login to the app', + description='Login fails with 500 error.', + type='INCIDENT', + original_type='Bug', + status='DONE', + original_status='Resolved', + created_date='2024-01-10T09:00:00Z', + updated_date='2024-01-15T14:00:00Z', + resolution_date='2024-01-15T14:00:00Z', + due_date=datetime.datetime(2024, 1, 20), + creator_id='10', + creator_name='John Smith', + assignee_id='11', + assignee_name='Jane Doe', + parent_issue_id=domain_id(Issue, context.connection.id, 40), + is_subtask=True, + priority='High', + component='Authentication', + lead_time_minutes=7500, + original_project='Test Project', + original_estimate_minutes=240, + time_spent_minutes=150, + time_remaining_minutes=90, + fix_versions='1.0.0', + ) + expected = [ + i, + ticket.BoardIssue(board_id=context.scope.domain_id(), issue_id=i.id), + ticket.IssueAssignee(issue_id=i.id, assignee_id='11', assignee_name='Jane Doe'), + ticket.IssueLabel(issue_id=issue_id, label_name='Authentication'), + ticket.IssueRelationship( + source_issue_id=domain_id(Issue, context.connection.id, 42), + target_issue_id=domain_id(Issue, context.connection.id, 43), + original_type='blocks', + ), + ] + assert_stream_convert(RedminePlugin, 'issues', raw, expected, context) + + +def test_issues_stream_due_date_as_string(context): + raw = { + 'id': 55, + 'project': {'id': 1, 'name': 'Test Project'}, + 'tracker': {'id': 2, 'name': 'Feature'}, + 'status': {'id': 1, 'name': 'New'}, + 'priority': {'id': 3, 'name': 'Normal'}, + 'author': {'id': 10, 'name': 'John Smith'}, + 'subject': 'Task with due date', + 'due_date': '2024-06-15', + 'created_on': '2024-02-01T10:00:00Z', + 'updated_on': '2024-02-01T10:00:00Z', + 'x_request_url': 'http://redmine.example.com/issues.json', + 'x_request_input': {'project_id': '1'}, + } + + stream = RedminePlugin().get_stream('issues') + tool_model = stream.extract(raw) + tool_model.connection_id = context.connection.id + tool_model.due_date = '2024-06-15' + + domain_models = list(stream.convert(tool_model, context)) + assert domain_models[0].due_date == datetime.datetime(2024, 6, 15) + + +def test_issues_stream_no_assignee(context): + raw = { + 'id': 10, + 'project': {'id': 1, 'name': 'Test Project'}, + 'tracker': {'id': 2, 'name': 'Feature'}, + 'status': {'id': 1, 'name': 'New'}, + 'priority': {'id': 3, 'name': 'Normal'}, + 'author': {'id': 10, 'name': 'John Smith'}, + 'subject': 'Add dark mode', + 'description': None, + 'start_date': None, + 'due_date': None, + 'done_ratio': 0, + 'estimated_hours': None, + 'created_on': '2024-02-01T10:00:00Z', + 'updated_on': '2024-02-01T10:00:00Z', + 'closed_on': None, + 'x_request_url': 'http://redmine.example.com/issues.json', + 'x_request_input': {'project_id': '1'}, + } + + i = ticket.Issue( + url='http://redmine.example.com/issues/10', + issue_key='10', + title='Add dark mode', + description=None, + type='REQUIREMENT', + original_type='Feature', + status='TODO', + original_status='New', + created_date='2024-02-01T10:00:00Z', + updated_date='2024-02-01T10:00:00Z', + resolution_date=None, + creator_id='10', + creator_name='John Smith', + priority='Normal', + original_project='Test Project', + ) + expected = [i, ticket.BoardIssue(board_id=context.scope.domain_id(), issue_id=i.id)] + assert_stream_convert(RedminePlugin, 'issues', raw, expected, context) + + +@pytest.mark.parametrize('tracker_name,expected_type', [ + ('Bug', 'INCIDENT'), + ('Feature', 'REQUIREMENT'), + ('Custom', 'TASK'), +]) +def test_issues_type_mapping_defaults(context, tracker_name, expected_type): + raw = _issue_raw(1, tracker_name, 'New') + expected = _expected_issue(context, 1, tracker_name, 'New', expected_type, 'TODO') + assert_stream_convert(RedminePlugin, 'issues', raw, expected, context) + + +@pytest.mark.parametrize('status_name,expected_status', [ + ('New', 'TODO'), + ('Closed', 'DONE'), + ('On Hold', 'OTHER'), +]) +def test_issues_status_mapping_defaults(context, status_name, expected_status): + raw = _issue_raw(2, 'Task', status_name) + expected = _expected_issue(context, 2, 'Task', status_name, 'TASK', expected_status) + assert_stream_convert(RedminePlugin, 'issues', raw, expected, context) + + +def test_issues_type_mapping_from_scope_config(): + context = ( + ContextBuilder(RedminePlugin()) + .with_connection(endpoint='http://redmine.example.com', token='test_api_key') + .with_scope('1', name='Test Project') + .with_scope_config(type_mappings={'Defect': TypeMapping(standard_type='BUG')}) + .build() + ) + raw = _issue_raw(3, 'Defect', 'New') + expected = _expected_issue(context, 3, 'Defect', 'New', 'BUG', 'TODO') + assert_stream_convert(RedminePlugin, 'issues', raw, expected, context) + + +def test_issues_status_mapping_from_scope_config(): + context = ( + ContextBuilder(RedminePlugin()) + .with_connection(endpoint='http://redmine.example.com', token='test_api_key') + .with_scope('1', name='Test Project') + .with_scope_config(status_mappings={'En cours': StatusMapping(standard_status='IN_PROGRESS')}) + .build() + ) + raw = _issue_raw(4, 'Task', 'En cours') + expected = _expected_issue(context, 4, 'Task', 'En cours', 'TASK', 'IN_PROGRESS') + assert_stream_convert(RedminePlugin, 'issues', raw, expected, context) + + +def test_issues_stream_custom_mappings(): + context = ( + ContextBuilder(RedminePlugin()) + .with_connection(endpoint='http://redmine.example.com', token='test_api_key') + .with_scope('1', name='Test Project') + .with_scope_config( + type_mappings={'Defect': {'standardType': 'BUG'}}, + status_mappings={'Ouvert': {'standardStatus': 'TODO'}}, + ) + .build() + ) + raw = _issue_raw(99, 'Defect', 'Ouvert', subject='Broken button') + expected = _expected_issue(context, 99, 'Defect', 'Ouvert', 'BUG', 'TODO', subject='Broken button') + assert_stream_convert(RedminePlugin, 'issues', raw, expected, context) + + +def test_issue_journals_stream(context_with_journals): + raw = { + 'id': 5, + '_issue_id': 42, + 'user': {'id': 11, 'name': 'Jane Doe'}, + 'notes': 'Investigated - network issue found.', + 'created_on': '2024-01-12T11:00:00Z', + 'private_notes': False, + 'details': [], + 'x_request_url': 'http://redmine.example.com/issues/42.json', + } + + expected = ticket.IssueComment( + issue_id=domain_id(Issue, context_with_journals.connection.id, 42), + body='Investigated - network issue found.', + account_id='11', + created_date='2024-01-12T11:00:00Z', + ) + assert_stream_convert(RedminePlugin, 'issuejournals', raw, expected, context_with_journals) + + +def test_issue_journals_changelog_stream(context_with_journals): + raw = { + 'id': 6, + '_issue_id': 42, + 'user': {'id': 11, 'name': 'Jane Doe'}, + 'notes': '', + 'created_on': '2024-01-13T11:00:00Z', + 'private_notes': False, + 'details': [ + { + 'property': 'attr', + 'name': 'assigned_to_id', + 'old_value': '10', + 'new_value': '11', + }, + { + 'property': 'attr', + 'name': 'status_id', + 'old_value': '1', + 'new_value': '2', + }, + ], + 'x_request_url': 'http://redmine.example.com/issues/42.json', + } + + issue_id = domain_id(Issue, context_with_journals.connection.id, 42) + expected = [ + ticket.IssueChangelogs( + id=domain_id(Journal, context_with_journals.connection.id, 6, 0), + issue_id=issue_id, + author_id='11', + author_name='Jane Doe', + field_id='assigned_to_id', + field_name='assignee', + original_from_value='10', + original_to_value='11', + from_value='10', + to_value='11', + created_date='2024-01-13T11:00:00Z', + ), + ticket.IssueChangelogs( + id=domain_id(Journal, context_with_journals.connection.id, 6, 1), + issue_id=issue_id, + author_id='11', + author_name='Jane Doe', + field_id='status_id', + field_name='status', + original_from_value='1', + original_to_value='2', + from_value='1', + to_value='2', + created_date='2024-01-13T11:00:00Z', + ), + ] + assert_stream_convert(RedminePlugin, 'issuejournals', raw, expected, context_with_journals) + + +def test_time_entries_stream(context): + raw = { + 'id': 7, + 'project': {'id': 1, 'name': 'Test Project'}, + 'issue': {'id': 42}, + 'user': {'id': 10, 'name': 'John Smith'}, + 'hours': 2.5, + 'comments': 'Fixed the login bug.', + 'spent_on': '2024-01-14', + 'created_on': '2024-01-14T17:00:00Z', + 'updated_on': '2024-01-14T17:00:00Z', + 'x_request_url': 'http://redmine.example.com/time_entries.json', + } + + expected = ticket.IssueWorklog( + issue_id=domain_id(Issue, context.connection.id, 42), + author_id='10', + comment='Fixed the login bug.', + time_spent_minutes=150, + started_date='2024-01-14', + logged_date='2024-01-14T17:00:00Z', + ) + assert_stream_convert(RedminePlugin, 'timeentries', raw, expected, context) + + +def test_time_entries_stream_no_issue(context): + raw = { + 'id': 8, + 'project': {'id': 1, 'name': 'Test Project'}, + 'user': {'id': 10, 'name': 'John Smith'}, + 'hours': 1.0, + 'comments': 'General project work', + 'spent_on': '2024-01-15', + 'created_on': '2024-01-15T18:00:00Z', + 'updated_on': '2024-01-15T18:00:00Z', + 'x_request_url': 'http://redmine.example.com/time_entries.json', + } + + assert_stream_convert(RedminePlugin, 'timeentries', raw, [], context) diff --git a/backend/python/pydevlake/pydevlake/collection_time.py b/backend/python/pydevlake/pydevlake/collection_time.py new file mode 100644 index 00000000000..b7863491111 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/collection_time.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from typing import Optional, Union + +from pydevlake.context import Context + + +def parse_time_after(value: Union[str, datetime.datetime, None]) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, datetime.datetime): + if value.tzinfo is None: + return value.replace(tzinfo=datetime.timezone.utc) + return value + text = str(value).strip() + if not text: + return None + if text.endswith('Z'): + text = text[:-1] + '+00:00' + return datetime.datetime.fromisoformat(text) + + +def collection_time_after(context: Context) -> Optional[datetime.datetime]: + if context.incremental: + return None + return context.time_after + + +def format_rfc3339_utc(value: datetime.datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.timezone.utc) + else: + value = value.astimezone(datetime.timezone.utc) + return value.strftime('%Y-%m-%dT%H:%M:%SZ') + + +def format_collection_date(value: datetime.datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.timezone.utc) + else: + value = value.astimezone(datetime.timezone.utc) + return value.strftime('%Y-%m-%d') diff --git a/backend/python/pydevlake/pydevlake/context.py b/backend/python/pydevlake/pydevlake/context.py index ad815da3c92..435dcc95387 100644 --- a/backend/python/pydevlake/pydevlake/context.py +++ b/backend/python/pydevlake/pydevlake/context.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime +from typing import Optional from sqlalchemy.engine import Engine @@ -25,12 +27,14 @@ def __init__(self, connection: Connection, scope: ToolScope, scope_config: ScopeConfig, - options: dict = None): + options: dict = None, + time_after: Optional[datetime.datetime] = None): self.engine = engine self.connection = connection self.scope = scope self.scope_config = scope_config self.options = options or {} + self.time_after = time_after self._engine = None @property diff --git a/backend/python/pydevlake/pydevlake/domain_layer/ticket.py b/backend/python/pydevlake/pydevlake/domain_layer/ticket.py new file mode 100644 index 00000000000..2eabf05c030 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/domain_layer/ticket.py @@ -0,0 +1,154 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from typing import Optional + +from sqlmodel import Field + +from pydevlake.model import DomainModel, DomainScope, NoPKModel + + +class Board(DomainScope, table=True): + __tablename__ = 'boards' + name: str + description: Optional[str] + url: Optional[str] + created_date: Optional[datetime] + type: Optional[str] + + +class BoardIssue(NoPKModel, table=True): + __tablename__ = 'board_issues' + board_id: str = Field(primary_key=True) + issue_id: str = Field(primary_key=True) + + +class BoardSprint(NoPKModel, table=True): + __tablename__ = 'board_sprints' + board_id: str = Field(primary_key=True) + sprint_id: str = Field(primary_key=True) + + +class Sprint(DomainModel, table=True): + __tablename__ = 'sprints' + name: str + url: Optional[str] + status: Optional[str] + started_date: Optional[datetime] + ended_date: Optional[datetime] + completed_date: Optional[datetime] + original_board_id: Optional[str] + + +class SprintIssue(NoPKModel, table=True): + __tablename__ = 'sprint_issues' + sprint_id: str = Field(primary_key=True) + issue_id: str = Field(primary_key=True) + + +class Issue(DomainModel, table=True): + __tablename__ = 'issues' + url: Optional[str] + issue_key: Optional[str] + title: str + description: Optional[str] + epic_key: Optional[str] + type: Optional[str] + original_type: Optional[str] + status: str + original_status: Optional[str] + story_point: Optional[float] + resolution_date: Optional[datetime] + created_date: Optional[datetime] + updated_date: Optional[datetime] + lead_time_minutes: Optional[int] + original_estimate_minutes: Optional[int] + time_spent_minutes: Optional[int] + time_remaining_minutes: Optional[int] + creator_id: Optional[str] + creator_name: Optional[str] + assignee_id: Optional[str] + assignee_name: Optional[str] + parent_issue_id: Optional[str] + priority: Optional[str] + severity: Optional[str] + urgency: Optional[str] + component: Optional[str] + icon_url: Optional[str] + original_project: Optional[str] + is_subtask: bool = False + due_date: Optional[datetime] + fix_versions: Optional[str] + + +class IssueAssignee(NoPKModel, table=True): + __tablename__ = 'issue_assignees' + issue_id: str = Field(primary_key=True) + assignee_id: str = Field(primary_key=True) + assignee_name: Optional[str] + + +class IssueLabel(NoPKModel, table=True): + __tablename__ = 'issue_labels' + issue_id: str = Field(primary_key=True) + label_name: str = Field(primary_key=True) + + +class IssueComment(DomainModel, table=True): + __tablename__ = 'issue_comments' + issue_id: str + body: Optional[str] + account_id: Optional[str] + created_date: Optional[datetime] + updated_date: Optional[datetime] + + +class IssueChangelogs(DomainModel, table=True): + __tablename__ = 'issue_changelogs' + issue_id: str + author_id: Optional[str] + author_name: Optional[str] + field_id: Optional[str] + field_name: Optional[str] + original_from_value: Optional[str] + original_to_value: Optional[str] + from_value: Optional[str] + to_value: Optional[str] + created_date: Optional[datetime] + + +class IssueWorklog(DomainModel, table=True): + __tablename__ = 'issue_worklogs' + author_id: Optional[str] + comment: Optional[str] + time_spent_minutes: Optional[int] + logged_date: Optional[datetime] + started_date: Optional[datetime] + issue_id: Optional[str] + + +class IssueRelationship(NoPKModel, table=True): + __tablename__ = 'issue_relationships' + source_issue_id: str = Field(primary_key=True) + target_issue_id: str = Field(primary_key=True) + original_type: str = Field(primary_key=True) + + +class IssueCustomArrayField(NoPKModel, table=True): + __tablename__ = 'issue_custom_array_fields' + issue_id: str = Field(primary_key=True) + field_id: str = Field(primary_key=True) + field_value: str = Field(primary_key=True) diff --git a/backend/python/pydevlake/pydevlake/ipc.py b/backend/python/pydevlake/pydevlake/ipc.py index 135d41cc1ad..12d03fbdf0e 100644 --- a/backend/python/pydevlake/pydevlake/ipc.py +++ b/backend/python/pydevlake/pydevlake/ipc.py @@ -24,6 +24,7 @@ from sqlmodel import create_engine from sqlalchemy.engine import Engine +from pydevlake.collection_time import parse_time_after from pydevlake.context import Context from pydevlake.message import Message from pydevlake.model import SubtaskRun @@ -122,7 +123,9 @@ def _mk_context(self, data: dict): scope_config_dict = data.get('scope_config') or {} scope_config = self._plugin.scope_config_type(**scope_config_dict) options = data.get('options', {}) - return Context(create_db_engine(db_url), connection, scope, scope_config, options) + sync_policy = data.get('sync_policy') or {} + time_after = parse_time_after(sync_policy.get('timeAfter')) + return Context(create_db_engine(db_url), connection, scope, scope_config, options, time_after) def create_db_engine(db_url) -> Engine: # SQLAlchemy doesn't understand postgres:// scheme diff --git a/backend/python/pydevlake/pydevlake/testing/testing.py b/backend/python/pydevlake/pydevlake/testing/testing.py index 3b1ec961706..8b9b7b39432 100644 --- a/backend/python/pydevlake/pydevlake/testing/testing.py +++ b/backend/python/pydevlake/pydevlake/testing/testing.py @@ -35,6 +35,8 @@ def __init__(self, plugin: Plugin): self.connection = None self.scope = None self.scope_config = None + self.options = None + self.time_after = None def with_connection(self, id=1, name='test_connection', **kwargs): self.connection = self.plugin.connection_type(id=id, name=name, **kwargs) @@ -50,16 +52,32 @@ def with_scope_config(self, id=1, name='test_config', **kwargs): self.scope_config = self.plugin.scope_config_type(id=id, name=name, **kwargs) return self + def with_options(self, **kwargs): + self.options = kwargs + return self + + def with_time_after(self, time_after): + self.time_after = time_after + return self + def build(self): - return make_context(self.connection, self.scope, self.scope_config) + return make_context( + self.connection, + self.scope, + self.scope_config, + options=self.options, + time_after=self.time_after, + ) -def make_context(connection, scope, scope_config): +def make_context(connection, scope, scope_config, options=None, time_after=None): return Context( engine=create_engine('sqlite:///:memory:'), scope=scope, connection=connection, - scope_config=scope_config + scope_config=scope_config, + options=options, + time_after=time_after, ) diff --git a/backend/python/pydevlake/tests/collection_time_test.py b/backend/python/pydevlake/tests/collection_time_test.py new file mode 100644 index 00000000000..cbccf029945 --- /dev/null +++ b/backend/python/pydevlake/tests/collection_time_test.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +from pydevlake.collection_time import collection_time_after, parse_time_after +from pydevlake.context import Context + + +def test_collection_time_after_uses_sync_policy(): + context = Context( + engine=None, + connection=None, + scope=None, + scope_config=None, + time_after=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ) + assert collection_time_after(context) == datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + + +def test_collection_time_after_skipped_when_incremental(): + context = Context( + engine=None, + connection=None, + scope=None, + scope_config=None, + options={'incremental': True}, + time_after=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ) + assert collection_time_after(context) is None + + +def test_parse_time_after_accepts_rfc3339(): + parsed = parse_time_after('2024-06-01T00:00:00Z') + assert parsed == datetime.datetime(2024, 6, 1, tzinfo=datetime.timezone.utc) diff --git a/backend/server/services/remote/bridge/bridge.go b/backend/server/services/remote/bridge/bridge.go index 825eef4ef0f..8a8f9483bd7 100644 --- a/backend/server/services/remote/bridge/bridge.go +++ b/backend/server/services/remote/bridge/bridge.go @@ -19,6 +19,7 @@ package bridge import ( "github.com/apache/devlake/core/errors" + coreModels "github.com/apache/devlake/core/models" "github.com/apache/devlake/core/plugin" "github.com/apache/devlake/server/services/remote/models" ) @@ -37,9 +38,23 @@ func NewBridge(invoker Invoker) *Bridge { return &Bridge{invoker: invoker} } +func enrichRemoteTaskData(ctx plugin.SubTaskContext) interface{} { + data := ctx.GetData() + taskData, ok := data.(models.RemotePluginTaskData) + if !ok { + return data + } + syncPolicy := ctx.TaskContext().SyncPolicy() + if syncPolicy == nil { + syncPolicy = &coreModels.SyncPolicy{} + } + models.ApplySyncPolicy(&taskData, syncPolicy) + return taskData +} + func (b *Bridge) RemoteSubtaskEntrypointHandler(subtaskMeta models.SubtaskMeta) plugin.SubTaskEntryPoint { return func(ctx plugin.SubTaskContext) errors.Error { - args := []interface{}{ctx.GetData()} + args := []interface{}{enrichRemoteTaskData(ctx)} for _, arg := range subtaskMeta.Arguments { args = append(args, arg) } diff --git a/backend/server/services/remote/models/task_data.go b/backend/server/services/remote/models/task_data.go new file mode 100644 index 00000000000..50909060235 --- /dev/null +++ b/backend/server/services/remote/models/task_data.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + coreModels "github.com/apache/devlake/core/models" +) + +type RemoteSyncPolicy struct { + TimeAfter *time.Time `json:"timeAfter,omitempty"` +} + +type RemotePluginTaskData struct { + DbUrl string `json:"db_url"` + Scope interface{} `json:"scope"` + Connection interface{} `json:"connection"` + ScopeConfig interface{} `json:"scope_config"` + Options map[string]interface{} `json:"options"` + SyncPolicy *RemoteSyncPolicy `json:"sync_policy,omitempty"` +} + +func ApplySyncPolicy(data *RemotePluginTaskData, syncPolicy *coreModels.SyncPolicy) { + if data == nil || syncPolicy == nil { + return + } + if data.Options == nil { + data.Options = make(map[string]interface{}) + } + if syncPolicy.FullSync { + data.Options["incremental"] = false + } + if syncPolicy.TimeAfter != nil { + data.SyncPolicy = &RemoteSyncPolicy{TimeAfter: syncPolicy.TimeAfter} + } +} diff --git a/backend/server/services/remote/models/task_data_test.go b/backend/server/services/remote/models/task_data_test.go new file mode 100644 index 00000000000..8375f1654ae --- /dev/null +++ b/backend/server/services/remote/models/task_data_test.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "testing" + "time" + + coreModels "github.com/apache/devlake/core/models" + "github.com/stretchr/testify/assert" +) + +func TestApplySyncPolicySetsIncrementalOnlyForFullSync(t *testing.T) { + timeAfter := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + t.Run("normal sync does not force incremental", func(t *testing.T) { + data := &RemotePluginTaskData{} + ApplySyncPolicy(data, &coreModels.SyncPolicy{TimeAfter: &timeAfter}) + _, hasIncremental := data.Options["incremental"] + assert.False(t, hasIncremental) + assert.NotNil(t, data.SyncPolicy) + assert.Equal(t, &timeAfter, data.SyncPolicy.TimeAfter) + }) + + t.Run("full sync forces non-incremental", func(t *testing.T) { + data := &RemotePluginTaskData{} + ApplySyncPolicy(data, &coreModels.SyncPolicy{ + TimeAfter: &timeAfter, + TriggerSyncPolicy: coreModels.TriggerSyncPolicy{ + FullSync: true, + }, + }) + incremental, ok := data.Options["incremental"].(bool) + assert.True(t, ok) + assert.False(t, incremental) + }) +} diff --git a/backend/server/services/remote/plugin/plugin_impl.go b/backend/server/services/remote/plugin/plugin_impl.go index 1cf768305d3..55d44555d72 100644 --- a/backend/server/services/remote/plugin/plugin_impl.go +++ b/backend/server/services/remote/plugin/plugin_impl.go @@ -48,13 +48,6 @@ type ( openApiSpec string connHelper *api.ConnectionApiHelper } - RemotePluginTaskData struct { - DbUrl string `json:"db_url"` - Scope interface{} `json:"scope"` - Connection interface{} `json:"connection"` - ScopeConfig interface{} `json:"scope_config"` - Options map[string]interface{} `json:"options"` - } ) func newPlugin(info *models.PluginInfo, invoker bridge.Invoker) (*remotePluginImpl, errors.Error) { @@ -177,7 +170,7 @@ func (p *remotePluginImpl) PrepareTaskData(taskCtx plugin.TaskContext, options m return nil, err } - return RemotePluginTaskData{ + return models.RemotePluginTaskData{ DbUrl: dbUrl, Scope: scope, Connection: connection, diff --git a/config-ui/src/plugins/components/scope-config-form/index.tsx b/config-ui/src/plugins/components/scope-config-form/index.tsx index 7bc8e4f8902..c692ef6f973 100644 --- a/config-ui/src/plugins/components/scope-config-form/index.tsx +++ b/config-ui/src/plugins/components/scope-config-form/index.tsx @@ -39,6 +39,7 @@ import { GhCopilotTransformation } from '@/plugins/register/gh-copilot'; import { AsanaTransformation } from '@/plugins/register/asana'; import { ClickUpTransformation } from '@/plugins/register/clickup'; import { YoutrackTransformation } from '@/plugins/register/youtrack'; +import { RedmineTransformation } from '@/plugins/register/redmine'; import { DOC_URL } from '@/release'; import { operator } from '@/utils'; @@ -337,6 +338,14 @@ export const ScopeConfigForm = ({ setTransformation={setTransformation} /> )} + + {plugin === 'redmine' && ( + + )} diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 75b54b3fa33..56c884ad0a4 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -38,6 +38,7 @@ import { JiraConfig } from './jira'; import { KiroConfig } from './kiro'; import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; +import { RedmineConfig } from './redmine'; import { RootlyConfig } from './rootly'; import { SonarQubeConfig } from './sonarqube'; import { TAPDConfig } from './tapd'; @@ -72,6 +73,7 @@ export const pluginConfigs: IPluginConfig[] = [ KiroConfig, LinearConfig, PagerDutyConfig, + RedmineConfig, RootlyConfig, SlackConfig, TempoConfig, diff --git a/config-ui/src/plugins/register/redmine/assets/icon.svg b/config-ui/src/plugins/register/redmine/assets/icon.svg new file mode 100644 index 00000000000..c4fd4aa3708 --- /dev/null +++ b/config-ui/src/plugins/register/redmine/assets/icon.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + diff --git a/config-ui/src/plugins/register/redmine/config.tsx b/config-ui/src/plugins/register/redmine/config.tsx new file mode 100644 index 00000000000..d67ce22ed15 --- /dev/null +++ b/config-ui/src/plugins/register/redmine/config.tsx @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; +import { RedmineDataScope } from './data-scope'; + +export const RedmineConfig: IPluginConfig = { + plugin: 'redmine', + name: 'Redmine', + icon: ({ color }) => , + sort: 19.5, + isBeta: true, + connection: { + docLink: 'https://www.redmine.org/projects/redmine/wiki/Rest_api', + initialValues: {}, + fields: [ + 'name', + { + key: 'endpoint', + subLabel: 'Base URL of your Redmine instance, e.g. https://redmine.example.com', + }, + { + key: 'token', + label: 'API Key', + subLabel: 'From Redmine: My account → API access key', + }, + 'proxy', + ], + }, + dataScope: { + title: 'Projects', + render: ({ connectionId, disabledItems, selectedItems, onChangeSelectedItems }) => ( + + ), + }, + scopeConfig: { + entities: ['TICKET'], + transformation: { + collectJournals: false, + typeMappings: {}, + statusMappings: {}, + }, + }, +}; diff --git a/config-ui/src/plugins/register/redmine/data-scope.tsx b/config-ui/src/plugins/register/redmine/data-scope.tsx new file mode 100644 index 00000000000..9a97327678e --- /dev/null +++ b/config-ui/src/plugins/register/redmine/data-scope.tsx @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo, useState } from 'react'; +import { SearchOutlined } from '@ant-design/icons'; +import { Input, Space, Tag, message } from 'antd'; + +import API from '@/api'; +import { Block, Loading } from '@/components'; + +const ROOT_GROUP_ID = 'root'; + +interface ProjectScopeData { + id: string; + name?: string; + identifier?: string; +} + +interface ScopeItem { + id: string; + name: string; + data: ProjectScopeData; +} + +interface Props { + connectionId: ID; + disabledItems?: Array<{ id: ID }>; + selectedItems: ScopeItem[]; + onChangeSelectedItems: (items: ScopeItem[]) => void; +} + +export const RedmineDataScope = ({ + connectionId, + disabledItems = [], + selectedItems, + onChangeSelectedItems, +}: Props) => { + const [loading, setLoading] = useState(true); + const [items, setItems] = useState([]); + const [query, setQuery] = useState(''); + + const disabledIds = useMemo(() => new Set(disabledItems.map((it) => `${it.id}`)), [disabledItems]); + + useEffect(() => { + let cancelled = false; + + const loadProjects = async () => { + setLoading(true); + try { + const res = await API.scope.remote('redmine', connectionId, { groupId: ROOT_GROUP_ID }); + if (cancelled) { + return; + } + const scopes = (res.children ?? []) + .filter((it) => it.type === 'scope') + .map((it) => ({ + id: `${it.id}`, + name: it.name, + data: it.data as ProjectScopeData, + })); + setItems(scopes); + } catch (err: any) { + if (!cancelled) { + message.error(err?.response?.data?.message ?? 'Failed to load Redmine projects'); + } + } finally { + if (!cancelled) { + setLoading(false); + } + } + }; + + loadProjects(); + + return () => { + cancelled = true; + }; + }, [connectionId]); + + const filteredItems = useMemo(() => { + const term = query.trim().toLowerCase(); + if (!term) { + return items; + } + return items.filter((item) => { + const identifier = item.data?.identifier ?? ''; + return item.name.toLowerCase().includes(term) || identifier.toLowerCase().includes(term); + }); + }, [items, query]); + + return ( + <> + + + {selectedItems.length ? ( + selectedItems.map((item) => ( + onChangeSelectedItems(selectedItems.filter((it) => it.id !== item.id))} + > + {item.name || item.id} + + )) + ) : ( + Please select scope... + )} + + + + } + placeholder="Search projects…" + value={query} + onChange={(e) => setQuery(e.target.value)} + /> + {loading ? ( + + ) : ( +
+ + + + + + + + {filteredItems.map((item) => { + const checked = selectedItems.some((it) => it.id === item.id); + const disabled = disabledIds.has(item.id); + return ( + + + + + ); + })} + +
+ Project
+ { + if (e.target.checked) { + onChangeSelectedItems([...selectedItems, item]); + } else { + onChangeSelectedItems(selectedItems.filter((it) => it.id !== item.id)); + } + }} + /> + {item.name}
+ {!filteredItems.length &&
No projects found
} +
+ )} + + + ); +}; diff --git a/config-ui/src/plugins/register/redmine/index.ts b/config-ui/src/plugins/register/redmine/index.ts new file mode 100644 index 00000000000..5f16858cbe4 --- /dev/null +++ b/config-ui/src/plugins/register/redmine/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; +export * from './transformation'; diff --git a/config-ui/src/plugins/register/redmine/transformation.tsx b/config-ui/src/plugins/register/redmine/transformation.tsx new file mode 100644 index 00000000000..6052567e164 --- /dev/null +++ b/config-ui/src/plugins/register/redmine/transformation.tsx @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo, useState } from 'react'; +import { Checkbox, Form, Select } from 'antd'; + +import { Block } from '@/components'; + +enum StandardType { + Requirement = 'REQUIREMENT', + Bug = 'BUG', + Incident = 'INCIDENT', +} + +enum StandardStatus { + Todo = 'TODO', + InProgress = 'IN_PROGRESS', + Done = 'DONE', +} + +const DEFAULT_TRACKER_NAMES = ['Bug', 'Feature', 'Support', 'Task']; +const DEFAULT_STATUS_NAMES = ['New', 'In Progress', 'Feedback', 'Resolved', 'Closed', 'Rejected']; + +const OPTIONAL_COLLECT_FIELDS = [ + { + key: 'collectJournals', + label: 'Journals', + description: 'Issue notes as comments and field changes as issue changelogs', + defaultEnabled: false, + }, + { + key: 'collectTimeEntries', + label: 'Time entries', + description: 'Logged hours as issue worklogs. Requires View time entries permission in Redmine', + defaultEnabled: false, + }, +] as const; + +const isFieldEnabled = (transformation: any, key: string, defaultEnabled: boolean) => + transformation[key] ?? defaultEnabled; + +interface Props { + entities: string[]; + transformation: any; + setTransformation: React.Dispatch>; +} + +const namesForStandardType = (typeMappings: Record | undefined, standardType: StandardType) => + Object.entries(typeMappings ?? {}) + .filter(([, value]) => (value?.standardType ?? value?.standard_type) === standardType) + .map(([name]) => name); + +const namesForStandardStatus = (statusMappings: Record | undefined, standardStatus: StandardStatus) => + Object.entries(statusMappings ?? {}) + .filter(([, value]) => (value?.standardStatus ?? value?.standard_status) === standardStatus) + .map(([name]) => name); + +const buildTypeMappings = (requirements: string[], bugs: string[], incidents: string[]) => ({ + ...Object.fromEntries(requirements.map((name) => [name, { standardType: StandardType.Requirement }])), + ...Object.fromEntries(bugs.map((name) => [name, { standardType: StandardType.Bug }])), + ...Object.fromEntries(incidents.map((name) => [name, { standardType: StandardType.Incident }])), +}); + +const buildStatusMappings = (todo: string[], inProgress: string[], done: string[]) => ({ + ...Object.fromEntries(todo.map((name) => [name, { standardStatus: StandardStatus.Todo }])), + ...Object.fromEntries(inProgress.map((name) => [name, { standardStatus: StandardStatus.InProgress }])), + ...Object.fromEntries(done.map((name) => [name, { standardStatus: StandardStatus.Done }])), +}); + +const selectOptions = (defaults: string[], selected: string[]) => + [...new Set([...defaults, ...selected])].sort().map((name) => ({ label: name, value: name })); + +export const RedmineTransformation = ({ entities, transformation, setTransformation }: Props) => { + const [requirements, setRequirements] = useState([]); + const [bugs, setBugs] = useState([]); + const [incidents, setIncidents] = useState([]); + const [todoStatuses, setTodoStatuses] = useState([]); + const [inProgressStatuses, setInProgressStatuses] = useState([]); + const [doneStatuses, setDoneStatuses] = useState([]); + + useEffect(() => { + setRequirements(namesForStandardType(transformation.typeMappings, StandardType.Requirement)); + setBugs(namesForStandardType(transformation.typeMappings, StandardType.Bug)); + setIncidents(namesForStandardType(transformation.typeMappings, StandardType.Incident)); + setTodoStatuses(namesForStandardStatus(transformation.statusMappings, StandardStatus.Todo)); + setInProgressStatuses(namesForStandardStatus(transformation.statusMappings, StandardStatus.InProgress)); + setDoneStatuses(namesForStandardStatus(transformation.statusMappings, StandardStatus.Done)); + }, [transformation]); + + const trackerOptions = useMemo( + () => selectOptions(DEFAULT_TRACKER_NAMES, [...requirements, ...bugs, ...incidents]), + [requirements, bugs, incidents], + ); + + const statusOptions = useMemo( + () => selectOptions(DEFAULT_STATUS_NAMES, [...todoStatuses, ...inProgressStatuses, ...doneStatuses]), + [todoStatuses, inProgressStatuses, doneStatuses], + ); + + const updateTypeMappings = (nextRequirements: string[], nextBugs: string[], nextIncidents: string[]) => { + setTransformation({ + ...transformation, + typeMappings: buildTypeMappings(nextRequirements, nextBugs, nextIncidents), + }); + }; + + const updateStatusMappings = (nextTodo: string[], nextInProgress: string[], nextDone: string[]) => { + setTransformation({ + ...transformation, + statusMappings: buildStatusMappings(nextTodo, nextInProgress, nextDone), + }); + }; + + if (!entities.includes('TICKET')) { + return null; + } + + return ( + <> + +
+ + updateTypeMappings(requirements, value, incidents)} + /> + + + updateStatusMappings(value, inProgressStatuses, doneStatuses)} + /> + + + updateStatusMappings(todoStatuses, inProgressStatuses, value)} + /> + +
+ + + +
+ {OPTIONAL_COLLECT_FIELDS.map(({ key, label, description, defaultEnabled }) => ( +
+ + setTransformation({ + ...transformation, + [key]: e.target.checked, + }) + } + > + {label} + +
{description}
+
+ ))} +
+ + + ); +}; diff --git a/grafana/dashboards/mysql/Redmine.json b/grafana/dashboards/mysql/Redmine.json new file mode 100644 index 00000000000..1321bbf957f --- /dev/null +++ b/grafana/dashboards/mysql/Redmine.json @@ -0,0 +1,1241 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Redmine", + "url": "https://www.redmine.org/projects/redmine/wiki/Rest_api" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows basic project management metrics from Redmine.\n- Data Source Required: Redmine", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": "mysql", + "description": "1. Total number of issues created.\n2. Issues are filtered by creation time (time filter at the upper-right corner) and Redmine project (\"Choose Project\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n case\r\n when avg_m is null then null\r\n when avg_m < 60 then concat(round(avg_m), ' minute', if(round(avg_m) = 1, '', 's'))\r\n when avg_m < 1440 then concat(round(avg_m / 60, 1), ' hour', if(round(avg_m / 60, 1) = 1, '', 's'))\r\n when avg_m < 10080 then concat(round(avg_m / 1440, 1), ' day', if(round(avg_m / 1440, 1) = 1, '', 's'))\r\n when avg_m < 43200 then concat(round(avg_m / 10080, 1), ' week', if(round(avg_m / 10080, 1) = 1, '', 's'))\r\n else concat(round(avg_m / 43200, 1), ' month', if(round(avg_m / 43200, 1) = 1, '', 's'))\r\n end as value\r\nfrom (\r\n select avg(i.lead_time_minutes) as avg_m\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n) _stats", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n),\r\n_p80 as (\r\n select max(lead_time_minutes) as p80_m\r\n from _ranks\r\n where ranks <= 0.8\r\n)\r\nselect\r\n case\r\n when p80_m is null then null\r\n when p80_m < 60 then concat(round(p80_m), ' minute', if(round(p80_m) = 1, '', 's'))\r\n when p80_m < 1440 then concat(round(p80_m / 60, 1), ' hour', if(round(p80_m / 60, 1) = 1, '', 's'))\r\n when p80_m < 10080 then concat(round(p80_m / 1440, 1), ' day', if(round(p80_m / 1440, 1) = 1, '', 's'))\r\n when p80_m < 43200 then concat(round(p80_m / 10080, 1), ' week', if(round(p80_m / 10080, 1) = 1, '', 's'))\r\n else concat(round(p80_m / 43200, 1), ' month', if(round(p80_m / 43200, 1) = 1, '', 's'))\r\n end as value\r\nfrom _p80", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Y-axis unit scales automatically (minutes, hours, days, weeks, or months) based on the largest monthly average.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(i.lead_time_minutes) as avg_m\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n),\r\n_scale as (\r\n select max(avg_m) as max_m from _requirements\r\n)\r\nselect \r\n date_format(r.time,'%M %Y') as month,\r\n case\r\n when s.max_m is null then null\r\n when s.max_m < 60 then round(r.avg_m, 1)\r\n when s.max_m < 1440 then round(r.avg_m / 60, 1)\r\n when s.max_m < 10080 then round(r.avg_m / 1440, 1)\r\n when s.max_m < 43200 then round(r.avg_m / 10080, 1)\r\n else round(r.avg_m / 43200, 1)\r\n end as mean_lead_time\r\nfrom _requirements r\r\n cross join _scale s\r\norder by r.time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "datasource": "mysql", + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 15, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.51, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\nselect \r\n now() as time,\r\n lpad(\r\n case\r\n when lead_time_minutes < 60 then concat(round(lead_time_minutes), 'm')\r\n when lead_time_minutes < 1440 then concat(round(lead_time_minutes / 60, 1), 'h')\r\n when lead_time_minutes < 10080 then concat(round(lead_time_minutes / 1440, 1), 'd')\r\n when lead_time_minutes < 43200 then concat(round(lead_time_minutes / 10080, 1), 'w')\r\n else concat(round(lead_time_minutes / 43200, 1), 'mo')\r\n end,\r\n 6, ' '\r\n ) as metric,\r\n percent_rank() over (order by lead_time_minutes asc) as value\r\nfrom _ranks\r\norder by lead_time_minutes asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "current" + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "mysql", + "definition": "select concat(name, '--', id) from boards where id like 'redmine%'", + "hide": 0, + "includeAll": true, + "label": "Choose Project", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'redmine%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "mysql", + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "Redmine", + "uid": "redmine-dashboard", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Redmine.json b/grafana/dashboards/postgresql/Redmine.json new file mode 100644 index 00000000000..fe61ab7e536 --- /dev/null +++ b/grafana/dashboards/postgresql/Redmine.json @@ -0,0 +1,1301 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Redmine", + "url": "https://www.redmine.org/projects/redmine/wiki/Rest_api" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows basic project management metrics from Redmine.\n- Data Source Required: Redmine", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "1. Total number of issues created.\n2. Issues are filtered by creation time (time filter at the upper-right corner) and Redmine project (\"Choose Project\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Issues\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Delivered Issues\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN avg_m IS NULL THEN NULL WHEN avg_m < 60 THEN ROUND(avg_m) || ' minute' || CASE WHEN ROUND(avg_m) = 1 THEN '' ELSE 's' END WHEN avg_m < 1440 THEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || ' hour' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN avg_m < 10080 THEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) || ' day' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN avg_m < 43200 THEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) || ' week' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END ELSE ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) || ' month' || CASE WHEN ROUND(CAST(CAST(avg_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END END AS value FROM (SELECT AVG(i.lead_time_minutes) AS avg_m FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) AS _stats", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))), _p80 AS (SELECT MAX(lead_time_minutes) AS p80_m FROM _ranks WHERE ranks <= 0.8) SELECT CASE WHEN p80_m IS NULL THEN NULL WHEN p80_m < 60 THEN ROUND(p80_m) || ' minute' || CASE WHEN ROUND(p80_m) = 1 THEN '' ELSE 's' END WHEN p80_m < 1440 THEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || ' hour' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN p80_m < 10080 THEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) || ' day' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END WHEN p80_m < 43200 THEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) || ' week' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END ELSE ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) || ' month' || CASE WHEN ROUND(CAST(CAST(p80_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) = 1 THEN '' ELSE 's' END END AS value FROM _p80", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Y-axis unit scales automatically (minutes, hours, days, weeks, or months) based on the largest monthly average.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(i.lead_time_minutes) AS avg_m FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1), _scale AS (SELECT MAX(avg_m) AS max_m FROM _requirements) SELECT TO_CHAR(CAST(r.time AS TIMESTAMP), 'FMMonth YYYY') AS month, CASE WHEN s.max_m IS NULL THEN NULL WHEN s.max_m < 60 THEN ROUND(r.avg_m, 1) WHEN s.max_m < 1440 THEN ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) WHEN s.max_m < 10080 THEN ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) WHEN s.max_m < 43200 THEN ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) ELSE ROUND(CAST(CAST(r.avg_m AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) END AS mean_lead_time FROM _requirements AS r CROSS JOIN _scale AS s ORDER BY r.time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 15, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.51, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, LPAD(CASE WHEN lead_time_minutes < 60 THEN ROUND(lead_time_minutes) || 'm' WHEN lead_time_minutes < 1440 THEN ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || 'h' WHEN lead_time_minutes < 10080 THEN ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0) AS DECIMAL), 1) || 'd' WHEN lead_time_minutes < 43200 THEN ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(10080, 0) AS DECIMAL), 1) || 'w' ELSE ROUND(CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(43200, 0) AS DECIMAL), 1) || 'mo' END, 6, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_minutes ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "current" + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'redmine%'", + "hide": 0, + "includeAll": true, + "label": "Choose Project", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'redmine%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "Redmine", + "uid": "redmine-dashboard-pg", + "version": 2, + "weekStart": "" +}