diff --git a/kubernetes/aio/leaderelection/__init__.py b/kubernetes/aio/leaderelection/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kubernetes/aio/leaderelection/electionconfig.py b/kubernetes/aio/leaderelection/electionconfig.py new file mode 100644 index 0000000000..ad1b1e4ad0 --- /dev/null +++ b/kubernetes/aio/leaderelection/electionconfig.py @@ -0,0 +1,73 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable, Coroutine +from typing import Any + +from kubernetes.aio.leaderelection.resourcelock.baselock import BaseLock + + +class Config: + # Validate config, exit if an error is detected + + # onstarted_leading and onstopped_leading accept either coroutines or + # coroutine functions. Coroutines faciliate passing context, but coroutine + # functions can be simpler when passing context is not required. + # + # One example of when passing context is helpful is sharing the ApiClient + # used by the leader election, which can then be used for subsequent + # Kubernetes API operations upon onstopped_leading or onstopped_leading. + def __init__( + self, + lock: BaseLock, + lease_duration: float, + renew_deadline: float, + retry_period: float, + onstarted_leading: Callable[[], Coroutine[Any, Any, None]] + | Coroutine[Any, Any, None], + onstopped_leading: Callable[[], Coroutine[Any, Any, None]] + | Coroutine[Any, Any, None], + ) -> None: + self.jitter_factor = 1.2 + + if lock is None: + raise ValueError("lock cannot be None") + self.lock = lock + + if lease_duration <= renew_deadline: + raise ValueError("lease_duration must be greater than renew_deadline") + + if renew_deadline <= self.jitter_factor * retry_period: + raise ValueError( + "renewDeadline must be greater than retry_period*jitter_factor" + ) + + if lease_duration < 1: + raise ValueError("lease_duration must be greater than one") + + if renew_deadline < 1: + raise ValueError("renew_deadline must be greater than one") + + if retry_period < 1: + raise ValueError("retry_period must be greater than one") + + self.lease_duration = lease_duration + self.renew_deadline = renew_deadline + self.retry_period = retry_period + + if onstarted_leading is None: + raise ValueError("callback onstarted_leading cannot be None") + self.onstarted_leading = onstarted_leading + + self.onstopped_leading = onstopped_leading diff --git a/kubernetes/aio/leaderelection/leaderelection.py b/kubernetes/aio/leaderelection/leaderelection.py new file mode 100644 index 0000000000..1289692b92 --- /dev/null +++ b/kubernetes/aio/leaderelection/leaderelection.py @@ -0,0 +1,253 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import datetime +import json +import logging +import sys +import time +from http import HTTPStatus + +from kubernetes.aio.client.exceptions import ApiException +from kubernetes.aio.leaderelection.electionconfig import Config +from kubernetes.aio.leaderelection.leaderelectionrecord import ( + LeaderElectionRecord, +) + +logger = logging.getLogger(__name__) + +""" +This package implements leader election using an annotation in a Kubernetes +object. The onstarted_leading coroutine is run as a task, which is cancelled if +the leader lock is obtained and then lost. + +At first all candidates are considered followers. The one to create a lock or +update an existing lock first becomes the leader and remains so until it fails +to renew its lease. +""" + + +class LeaderElection: + def __init__(self, election_config: Config) -> None: + if election_config is None: + sys.exit("argument config not passed") + + # Latest record observed in the created lock object + self.observed_record: LeaderElectionRecord | None = None + + # The configuration set for this candidate + self.election_config: Config = election_config + + # Latest update time of the lock + self.observed_time_milliseconds = 0 + + # Point of entry to Leader election + async def run(self) -> None: + # Try to create/ acquire a lock + if await self.acquire(): + logger.info( + "%s successfully acquired lease", self.election_config.lock.identity + ) + + onstarted_leading_coroutine = ( + self.election_config.onstarted_leading() + if callable(self.election_config.onstarted_leading) + else self.election_config.onstarted_leading + ) + + task = asyncio.create_task(onstarted_leading_coroutine) + + await self.renew_loop() + + # Leader lock lost - cancel the onstarted_leading coroutine if it's + # still running. This permits onstarted_leading to clean up state + # that might not be accessible to onstopped_leading. + task.cancel() + + # Failed to update lease, run onstopped_leading callback. This is + # preserved in order to continue to provide an interface similar to + # the one provided by `kubernetes-client/python`. + if self.election_config.onstopped_leading is not None: + await ( + self.election_config.onstopped_leading() + if callable(self.election_config.onstopped_leading) + else self.election_config.onstopped_leading + ) + + async def acquire(self) -> bool: + # Follower + logger.debug("%s is a follower", self.election_config.lock.identity) + retry_period = self.election_config.retry_period + + while True: + succeeded = await self.try_acquire_or_renew() + + if succeeded: + return True + + await asyncio.sleep(retry_period) + + async def renew_loop(self) -> None: + # Leader + logger.debug( + "Leader has entered renew loop and will try to update lease continuously" + ) + + retry_period = self.election_config.retry_period + renew_deadline = self.election_config.renew_deadline * 1000 + + while True: + timeout = int(time.time() * 1000) + renew_deadline + succeeded = False + + while int(time.time() * 1000) < timeout: + succeeded = await self.try_acquire_or_renew() + + if succeeded: + break + await asyncio.sleep(retry_period) + + if succeeded: + await asyncio.sleep(retry_period) + continue + + # failed to renew, return + return + + async def try_acquire_or_renew(self) -> bool: + now_timestamp = time.time() + now = datetime.datetime.fromtimestamp(now_timestamp) + + # Check if lock is created + lock_status, old_election_record = await self.election_config.lock.get( + self.election_config.lock.name, self.election_config.lock.namespace + ) + + # create a default Election record for this candidate + leader_election_record = LeaderElectionRecord( + self.election_config.lock.identity, + str(self.election_config.lease_duration), + str(now), + str(now), + ) + + # A lock is not created with that name, try to create one + if not lock_status: + assert ( + isinstance(old_election_record, ApiException) + and old_election_record.body is not None + ) + if json.loads(old_election_record.body)["code"] != HTTPStatus.NOT_FOUND: + logger.error( + "Error retrieving resource lock %s as %s", + self.election_config.lock.name, + old_election_record.reason, + ) + return False + + logger.debug( + "%s is trying to create a lock", + leader_election_record.holder_identity, + ) + create_status = await self.election_config.lock.create( + name=self.election_config.lock.name, + namespace=self.election_config.lock.namespace, + election_record=leader_election_record, + ) + + if not create_status: + logger.error( + "%s failed to create lock", leader_election_record.holder_identity + ) + return False + + self.observed_record = leader_election_record + self.observed_time_milliseconds = int(time.time() * 1000) + return True + + # A lock exists with that name + # Validate old_election_record + if old_election_record is None: + # try to update lock with proper election record + return await self.update_lock(leader_election_record) + + assert isinstance(old_election_record, LeaderElectionRecord) + if ( + old_election_record.holder_identity is None + or old_election_record.lease_duration is None + or old_election_record.acquire_time is None + or old_election_record.renew_time is None + ): + # try to update lock with proper election record + return await self.update_lock(leader_election_record) + + # Report transitions + if ( + self.observed_record + and self.observed_record.holder_identity + != old_election_record.holder_identity + ): + logger.debug( + "Leader has switched to %s", old_election_record.holder_identity + ) + + if ( + self.observed_record is None + or old_election_record.__dict__ != self.observed_record.__dict__ + ): + self.observed_record = old_election_record + self.observed_time_milliseconds = int(time.time() * 1000) + + # If This candidate is not the leader and lease duration is yet to finish + if ( + self.election_config.lock.identity != self.observed_record.holder_identity + and self.observed_time_milliseconds + + self.election_config.lease_duration * 1000 + > int(now_timestamp * 1000) + ): + logger.debug( + "Yet to finish lease_duration, lease held by %s and has not expired", + old_election_record.holder_identity, + ) + return False + + # If this candidate is the Leader + if self.election_config.lock.identity == self.observed_record.holder_identity: + # Leader updates renewTime, but keeps acquire_time unchanged + leader_election_record.acquire_time = self.observed_record.acquire_time + + return await self.update_lock(leader_election_record) + + async def update_lock(self, leader_election_record: LeaderElectionRecord) -> bool: + # Update object with latest election record + update_status = await self.election_config.lock.update( + self.election_config.lock.name, + self.election_config.lock.namespace, + leader_election_record, + ) + + if not update_status: + logger.warning( + "%s failed to acquire lease", leader_election_record.holder_identity + ) + return False + + self.observed_record = leader_election_record + self.observed_time_milliseconds = int(time.time() * 1000) + logger.debug( + "Leader %s has successfully updated lease", + leader_election_record.holder_identity, + ) + return True diff --git a/kubernetes/aio/leaderelection/leaderelection_test.py b/kubernetes/aio/leaderelection/leaderelection_test.py new file mode 100644 index 0000000000..3d1e35dc0e --- /dev/null +++ b/kubernetes/aio/leaderelection/leaderelection_test.py @@ -0,0 +1,369 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import asyncio +import json +import unittest +from collections.abc import Callable +from unittest import IsolatedAsyncioTestCase + +from kubernetes.aio.client.rest import ApiException +from kubernetes.aio.leaderelection import electionconfig, leaderelection +from kubernetes.aio.leaderelection.leaderelectionrecord import ( + LeaderElectionRecord, +) +from kubernetes.aio.leaderelection.resourcelock.baselock import BaseLock + + +class LeaderElectionTest(IsolatedAsyncioTestCase): + async def test_simple_leader_election(self) -> None: + election_history = [] + leadership_history = [] + + def on_create(): + election_history.append("create record") + leadership_history.append("get leadership") + + def on_update(): + election_history.append("update record") + + def on_change(): + election_history.append("change record") + + mock_lock = MockResourceLock( + "mock", + "mock_namespace", + "mock", + asyncio.Lock(), + on_create, + on_update, + on_change, + None, + ) + + async def on_started_leading() -> None: + leadership_history.append("start leading") + + async def on_stopped_leading() -> None: + leadership_history.append("stop leading") + + # Create config 4.5 4 3 + config = electionconfig.Config( + lock=mock_lock, + lease_duration=2.5, + renew_deadline=2, + retry_period=1.5, + onstarted_leading=on_started_leading(), + onstopped_leading=on_stopped_leading(), + ) + + # Enter leader election + await leaderelection.LeaderElection(config).run() + + self.assert_history( + election_history, + ["create record", "update record", "update record", "update record"], + ) + self.assert_history( + leadership_history, ["get leadership", "start leading", "stop leading"] + ) + + async def test_leader_election(self) -> None: + election_history = [] + leadership_history = [] + + def on_create_A(): + election_history.append("A creates record") + leadership_history.append("A gets leadership") + + def on_update_A(): + election_history.append("A updates record") + + def on_change_A(): + election_history.append("A gets leadership") + + lock = asyncio.Lock() + + mock_lock_A = MockResourceLock( + "mock", + "mock_namespace", + "MockA", + lock, + on_create_A, + on_update_A, + on_change_A, + None, + ) + mock_lock_A.renew_count_max = 3 + + async def on_started_leading_A(): + leadership_history.append("A starts leading") + + async def on_stopped_leading_A(): + leadership_history.append("A stops leading") + + config_A = electionconfig.Config( + lock=mock_lock_A, + lease_duration=2.5, + renew_deadline=2, + retry_period=1.5, + onstarted_leading=on_started_leading_A(), + onstopped_leading=on_stopped_leading_A(), + ) + + def on_create_B(): + election_history.append("B creates record") + leadership_history.append("B gets leadership") + + def on_update_B(): + election_history.append("B updates record") + + def on_change_B(): + leadership_history.append("B gets leadership") + + mock_lock_B = MockResourceLock( + "mock", + "mock_namespace", + "MockB", + lock, + on_create_B, + on_update_B, + on_change_B, + None, + ) + mock_lock_B.renew_count_max = 4 + + async def on_started_leading_B(): + leadership_history.append("B starts leading") + + async def on_stopped_leading_B(): + leadership_history.append("B stops leading") + + config_B = electionconfig.Config( + lock=mock_lock_B, + lease_duration=2.5, + renew_deadline=2, + retry_period=1.5, + onstarted_leading=on_started_leading_B(), + onstopped_leading=on_stopped_leading_B(), + ) + + mock_lock_B.leader_record = mock_lock_A.leader_record + + config_A_election = asyncio.create_task( + leaderelection.LeaderElection(config_A).run() + ) + config_B_election = asyncio.create_task( + leaderelection.LeaderElection(config_B).run() + ) + + await asyncio.gather(config_A_election, config_B_election) + + self.assert_history( + election_history, + [ + "A creates record", + "A updates record", + "A updates record", + "B updates record", + "B updates record", + "B updates record", + "B updates record", + ], + ) + self.assert_history( + leadership_history, + [ + "A gets leadership", + "A starts leading", + "A stops leading", + "B gets leadership", + "B starts leading", + "B stops leading", + ], + ) + + """Expected behavior: to check if the leader stops leading if it fails to update the lock within the renew_deadline + and stops leading after finally timing out. The difference between each try comes out to be approximately the sleep + time. + Example: + create record: 0s + on try update: 1.5s + on update: zzz s + on try update: 3s + on update: zzz s + on try update: 4.5s + on try update: 6s + Timeout - Leader Exits""" + + async def test_leader_election_with_renew_deadline(self) -> None: + election_history = [] + leadership_history = [] + + def on_create(): + election_history.append("create record") + leadership_history.append("get leadership") + + def on_update(): + election_history.append("update record") + + def on_change(): + election_history.append("change record") + + def on_try_update(): + election_history.append("try update record") + + mock_lock = MockResourceLock( + "mock", + "mock_namespace", + "mock", + asyncio.Lock(), + on_create, + on_update, + on_change, + on_try_update, + ) + mock_lock.renew_count_max = 3 + + async def on_started_leading(): + leadership_history.append("start leading") + + async def on_stopped_leading(): + leadership_history.append("stop leading") + + # Create config + config = electionconfig.Config( + lock=mock_lock, + lease_duration=2.5, + renew_deadline=2, + retry_period=1.5, + onstarted_leading=on_started_leading(), + onstopped_leading=on_stopped_leading(), + ) + + # Enter leader election + await leaderelection.LeaderElection(config).run() + + self.assert_history( + election_history, + [ + "create record", + "try update record", + "update record", + "try update record", + "update record", + "try update record", + "try update record", + ], + ) + + self.assert_history( + leadership_history, ["get leadership", "start leading", "stop leading"] + ) + + def assert_history(self, history, expected) -> None: + self.assertIsNotNone(expected) + self.assertIsNotNone(history) + self.assertEqual(len(expected), len(history)) + + for idx in range(len(history)): + self.assertEqual( + history[idx], + expected[idx], + msg=f"Not equal at index {idx}, expected {expected[idx]}, got {history[idx]}", + ) + + +class MockResourceLock(BaseLock): + def __init__( + self, + name: str, + namespace: str, + identity: str, + shared_lock: asyncio.Lock, + on_create: Callable, + on_update: Callable, + on_change: Callable, + on_try_update: Callable | None, + ) -> None: + # self.leader_record is shared between two MockResourceLock objects + self.leader_record: list[LeaderElectionRecord] = [] + self.renew_count = 0 + self.renew_count_max = 4 + self.name = name + self.namespace = namespace + self.identity = str(identity) + self.lock = shared_lock + + self.on_create = on_create + self.on_update = on_update + self.on_change = on_change + self.on_try_update = on_try_update + + async def get( + self, name: str, namespace: str + ) -> tuple[bool, LeaderElectionRecord] | tuple[bool, Exception] | tuple[bool, None]: + await self.lock.acquire() + try: + if self.leader_record: + return True, self.leader_record[0] + + ex = ApiException() + ex.body = json.dumps({"code": 404}).encode() + return False, ex + finally: + self.lock.release() + + async def create( + self, name: str, namespace: str, election_record: LeaderElectionRecord + ) -> bool: + await self.lock.acquire() + try: + if len(self.leader_record) == 1: + return False + self.leader_record.append(election_record) + self.on_create() + self.renew_count += 1 + return True + finally: + self.lock.release() + + async def update( + self, name: str, namespace: str, updated_record: LeaderElectionRecord + ) -> bool: + await self.lock.acquire() + try: + if self.on_try_update: + self.on_try_update() + if self.renew_count >= self.renew_count_max: + return False + + old_record = self.leader_record[0] + self.leader_record[0] = updated_record + + self.on_update() + + if old_record.holder_identity != updated_record.holder_identity: + await asyncio.sleep(2) + self.on_change() + + self.renew_count += 1 + return True + finally: + self.lock.release() + + +if __name__ == "__main__": + unittest.main() diff --git a/kubernetes/base/hack/verify-boilerplate.sh b/kubernetes/aio/leaderelection/leaderelectionrecord.py old mode 100755 new mode 100644 similarity index 53% rename from kubernetes/base/hack/verify-boilerplate.sh rename to kubernetes/aio/leaderelection/leaderelectionrecord.py index 2f54c8cc38..f51f5363c5 --- a/kubernetes/base/hack/verify-boilerplate.sh +++ b/kubernetes/aio/leaderelection/leaderelectionrecord.py @@ -1,6 +1,4 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Kubernetes Authors. +# Copyright 2021 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,22 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. - -boilerDir="${KUBE_ROOT}/hack/boilerplate" -boiler="${boilerDir}/boilerplate.py" - -files_need_boilerplate=($(${boiler} "$@")) - -# Run boilerplate check -if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then - for file in "${files_need_boilerplate[@]}"; do - echo "Boilerplate header is wrong for: ${file}" >&2 - done - exit 1 -fi +class LeaderElectionRecord: + # Leader election details, used in the lock object + def __init__( + self, + holder_identity: str | None, + lease_duration: str | None, + acquire_time: str | None, + renew_time: str | None, + ): + self.holder_identity = holder_identity + self.lease_duration = lease_duration + self.acquire_time = acquire_time + self.renew_time = renew_time diff --git a/kubernetes/aio/leaderelection/resourcelock/__init__.py b/kubernetes/aio/leaderelection/resourcelock/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kubernetes/aio/leaderelection/resourcelock/baselock.py b/kubernetes/aio/leaderelection/resourcelock/baselock.py new file mode 100644 index 0000000000..65aee6a2e4 --- /dev/null +++ b/kubernetes/aio/leaderelection/resourcelock/baselock.py @@ -0,0 +1,62 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import abstractmethod + +from kubernetes.aio.leaderelection.leaderelectionrecord import ( + LeaderElectionRecord, +) + + +class BaseLock: + def __init__(self, name: str, namespace: str, identity: str) -> None: + self.name = name + self.namespace = namespace + self.identity = str(identity) + + # get returns the election record from a ConfigMap Annotation + @abstractmethod + async def get( + self, name: str, namespace: str + ) -> tuple[bool, LeaderElectionRecord] | tuple[bool, Exception] | tuple[bool, None]: + """ + :param name: Name of the configmap object information to get + :param namespace: Namespace in which the configmap object is to be searched + :return: 'True, election record' if object found else 'False, exception response' + """ + ... + + @abstractmethod + async def create( + self, name: str, namespace: str, election_record: LeaderElectionRecord + ) -> bool: + """ + :param electionRecord: Annotation string + :param name: Name of the configmap object to be created + :param namespace: Namespace in which the configmap object is to be created + :return: 'True' if object is created else 'False' if failed + """ + ... + + @abstractmethod + async def update( + self, name: str, namespace: str, updated_record: LeaderElectionRecord + ) -> bool: + """ + :param name: name of the lock to be updated + :param namespace: namespace the lock is in + :param updated_record: the updated election record + :return: True if update is successful False if it fails + """ + ... diff --git a/kubernetes/aio/leaderelection/resourcelock/configmaplock.py b/kubernetes/aio/leaderelection/resourcelock/configmaplock.py new file mode 100644 index 0000000000..53a46c09bf --- /dev/null +++ b/kubernetes/aio/leaderelection/resourcelock/configmaplock.py @@ -0,0 +1,166 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +from typing import Any + +from kubernetes.aio import client +from kubernetes.aio.client.api_client import ApiClient +from kubernetes.aio.client.rest import ApiException +from kubernetes.aio.leaderelection.leaderelectionrecord import ( + LeaderElectionRecord, +) +from kubernetes.aio.leaderelection.resourcelock.baselock import BaseLock + +logger = logging.getLogger(__name__) + + +class ConfigMapLock(BaseLock): + def __init__(self, name: str, namespace: str, identity: str, api_client: ApiClient): + """ + :param name: name of the lock + :param namespace: namespace + :param identity: A unique identifier that the candidate is using + """ + super().__init__(name, namespace, identity) + + # self._api_instance = None # See api_instance property + self.api_instance = client.CoreV1Api(api_client=api_client) + self.leader_electionrecord_annotationkey = ( + "control-plane.alpha.kubernetes.io/leader" + ) + self.configmap_reference: client.V1ConfigMap | None = None + self.lock_record: dict[str, Any] = { + "holderIdentity": None, + "leaseDurationSeconds": None, + "acquireTime": None, + "renewTime": None, + } + + # get returns the election record from a ConfigMap Annotation + async def get( + self, name: str, namespace: str + ) -> tuple[bool, LeaderElectionRecord] | tuple[bool, Exception] | tuple[bool, None]: + """ + :param name: Name of the configmap object information to get + :param namespace: Namespace in which the configmap object is to be searched + :return: 'True, election record' if object found else 'False, exception response' + """ + try: + api_response = await self.api_instance.read_namespaced_config_map( + name, namespace + ) + + # If an annotation does not exist - add the leader_electionrecord_annotationkey + annotations = api_response.metadata.annotations + if annotations is None or annotations == "": + api_response.metadata.annotations = { + self.leader_electionrecord_annotationkey: "" + } + self.configmap_reference = api_response + return True, None + + # If an annotation exists but, the leader_electionrecord_annotationkey does not then add it as a key + if not annotations.get(self.leader_electionrecord_annotationkey): + api_response.metadata.annotations = { + self.leader_electionrecord_annotationkey: "" + } + self.configmap_reference = api_response + return True, None + + lock_record = self.get_lock_object( + json.loads(annotations[self.leader_electionrecord_annotationkey]) + ) + + self.configmap_reference = api_response + return True, lock_record + except ApiException as e: + return False, e + + async def create( + self, name: str, namespace: str, election_record: LeaderElectionRecord + ) -> bool: + """ + :param electionRecord: Annotation string + :param name: Name of the configmap object to be created + :param namespace: Namespace in which the configmap object is to be created + :return: 'True' if object is created else 'False' if failed + """ + body = client.V1ConfigMap( + metadata=client.V1ObjectMeta( + name=name, + annotations={ + self.leader_electionrecord_annotationkey: json.dumps( + self.get_lock_dict(election_record) + ) + }, + ) + ) + + try: + await self.api_instance.create_namespaced_config_map( + namespace, body, pretty=True + ) + return True + except ApiException: + logger.exception("Failed to create lock") + return False + + async def update( + self, name: str, namespace: str, updated_record: LeaderElectionRecord + ) -> bool: + """ + :param name: name of the lock to be updated + :param namespace: namespace the lock is in + :param updated_record: the updated election record + :return: True if update is successful False if it fails + """ + try: + # Set the updated record + assert self.configmap_reference is not None + self.configmap_reference.metadata.annotations[ + self.leader_electionrecord_annotationkey + ] = json.dumps(self.get_lock_dict(updated_record)) + await self.api_instance.replace_namespaced_config_map( + name=name, namespace=namespace, body=self.configmap_reference + ) + return True + except ApiException: + logger.exception("Failed to update lock") + return False + + def get_lock_object(self, lock_record: dict) -> LeaderElectionRecord: + leader_election_record = LeaderElectionRecord(None, None, None, None) + + if lock_record.get("holderIdentity"): + leader_election_record.holder_identity = lock_record["holderIdentity"] + if lock_record.get("leaseDurationSeconds"): + leader_election_record.lease_duration = lock_record["leaseDurationSeconds"] + if lock_record.get("acquireTime"): + leader_election_record.acquire_time = lock_record["acquireTime"] + if lock_record.get("renewTime"): + leader_election_record.renew_time = lock_record["renewTime"] + + return leader_election_record + + def get_lock_dict( + self, leader_election_record: LeaderElectionRecord + ) -> dict[str, Any]: + self.lock_record["holderIdentity"] = leader_election_record.holder_identity + self.lock_record["leaseDurationSeconds"] = leader_election_record.lease_duration + self.lock_record["acquireTime"] = leader_election_record.acquire_time + self.lock_record["renewTime"] = leader_election_record.renew_time + + return self.lock_record diff --git a/kubernetes/aio/leaderelection/resourcelock/leaselock.py b/kubernetes/aio/leaderelection/resourcelock/leaselock.py new file mode 100644 index 0000000000..aad7c36a7d --- /dev/null +++ b/kubernetes/aio/leaderelection/resourcelock/leaselock.py @@ -0,0 +1,167 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from datetime import datetime + +from kubernetes.aio import client +from kubernetes.aio.client.api_client import ApiClient +from kubernetes.aio.client.rest import ApiException +from kubernetes.aio.leaderelection.leaderelectionrecord import ( + LeaderElectionRecord, +) +from kubernetes.aio.leaderelection.resourcelock.baselock import BaseLock + +logger = logging.getLogger(__name__) + + +class LeaseLock(BaseLock): + def __init__(self, name: str, namespace: str, identity: str, api_client: ApiClient): + """ + :param name: name of the lock + :param namespace: namespace + :param identity: A unique identifier that the candidate is using + """ + super().__init__(name, namespace, identity) + + self.api_instance = client.CoordinationV1Api(api_client=api_client) + + # lease resource identity and reference + self.lease_reference: client.V1Lease | None = None + + # get returns the election record from a Lease Annotation + async def get( + self, name: str, namespace: str + ) -> tuple[bool, LeaderElectionRecord] | tuple[bool, Exception] | tuple[bool, None]: + """ + :param name: Name of the lease object information to get + :param namespace: Namespace in which the lease object is to be searched + :return: 'True, election record' if object found else 'False, exception response' + """ + try: + lease = await self.api_instance.read_namespaced_lease(name, namespace) + except ApiException as e: + return False, e + else: + self.lease_reference = lease + return True, self.election_record(lease) + + async def create( + self, name: str, namespace: str, election_record: LeaderElectionRecord + ) -> bool: + """ + :param electionRecord: Annotation string + :param name: Name of the lease object to be created + :param namespace: Namespace in which the lease object is to be created + :return: 'True' if object is created else 'False' if failed + """ + body = client.V1Lease( + metadata=client.V1ObjectMeta(name=name), + spec=self.update_lease(election_record), + ) + + try: + await self.api_instance.create_namespaced_lease( + namespace, body, pretty=True + ) + return True + except ApiException: + logger.exception("Failed to create lock") + return False + + async def update( + self, name: str, namespace: str, updated_record: LeaderElectionRecord + ) -> bool: + """ + :param name: name of the lock to be updated + :param namespace: namespace the lock is in + :param updated_record: the updated election record + :return: True if update is successful False if it fails + """ + try: + # update the Lease from the updated record + assert self.lease_reference is not None + self.lease_reference.spec = self.update_lease( + updated_record, self.lease_reference.spec + ) + + await self.api_instance.replace_namespaced_lease( + name=name, namespace=namespace, body=self.lease_reference + ) + return True + except ApiException: + logger.exception("Failed to update lock") + return False + + def update_lease( + self, + leader_election_record: LeaderElectionRecord, + current_spec: client.V1LeaseSpec | None = None, + ): + # existing or new lease? + spec = current_spec if current_spec else client.V1LeaseSpec() + + # lease configuration + assert leader_election_record.holder_identity + spec.holder_identity = leader_election_record.holder_identity + + assert leader_election_record.lease_duration + spec.lease_duration_seconds = int(leader_election_record.lease_duration) + + acquire_time = self.time_str_to_iso(leader_election_record.acquire_time) + if acquire_time: + spec.acquire_time = acquire_time + + renew_time = self.time_str_to_iso(leader_election_record.renew_time) + if renew_time: + spec.renew_time = renew_time + + return spec + + def election_record(self, lease: client.V1Lease): + """ + Get leader election record from Lease spec. + """ + leader_election_record = LeaderElectionRecord(None, None, None, None) + + if not lease.spec: + return leader_election_record + + if lease.spec.holder_identity: + leader_election_record.holder_identity = lease.spec.holder_identity + if lease.spec.lease_duration_seconds: + leader_election_record.lease_duration = str( + lease.spec.lease_duration_seconds + ) + if lease.spec.acquire_time: + leader_election_record.acquire_time = str( + datetime.replace(lease.spec.acquire_time, tzinfo=None) + ) + if lease.spec.renew_time: + leader_election_record.renew_time = str( + datetime.replace(lease.spec.renew_time, tzinfo=None) + ) + + return leader_election_record + + # conversion between kubernetes ISO formatted time and elector record time + def time_str_to_iso(self, str_time) -> str | None: + formats = ["%Y-%m-%d %H:%M:%S.%f%z", "%Y-%m-%d %H:%M:%S.%f"] + for fmt in formats: + try: + return datetime.strptime(str_time, fmt).isoformat() + "Z" + except ValueError: + pass + logger.error("Failed to parse time string: %s", str_time) + return None diff --git a/kubernetes/base/.github/PULL_REQUEST_TEMPLATE.md b/kubernetes/base/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index f6af35b429..0000000000 --- a/kubernetes/base/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,72 +0,0 @@ - - -#### What type of PR is this? - - - -#### What this PR does / why we need it: - -#### Which issue(s) this PR fixes: - -Fixes # - -#### Special notes for your reviewer: - -#### Does this PR introduce a user-facing change? - -```release-note - -``` - -#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.: - - -```docs - -``` diff --git a/kubernetes/base/.gitignore b/kubernetes/base/.gitignore deleted file mode 100644 index 3054962009..0000000000 --- a/kubernetes/base/.gitignore +++ /dev/null @@ -1,95 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# IPython Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env - -# virtualenv -venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject - -# Intellij IDEA files -.idea/* -*.iml -.vscode - diff --git a/kubernetes/base/.travis.yml b/kubernetes/base/.travis.yml deleted file mode 100644 index 86a1bfa2ad..0000000000 --- a/kubernetes/base/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -# ref: https://docs.travis-ci.com/user/languages/python -language: python -dist: xenial - -stages: - - verify boilerplate - - test - -install: - - pip install tox - -script: - - ./run_tox.sh tox - -jobs: - include: - - stage: verify boilerplate - script: ./hack/verify-boilerplate.sh - python: 3.7 - - stage: test - python: 3.9 - env: TOXENV=update-pycodestyle - - python: 3.9 - env: TOXENV=coverage,codecov - - python: 3.7 - env: TOXENV=docs - - python: 3.5 - env: TOXENV=py35 - - python: 3.5 - env: TOXENV=py35-functional - - python: 3.6 - env: TOXENV=py36 - - python: 3.6 - env: TOXENV=py36-functional - - python: 3.7 - env: TOXENV=py37 - - python: 3.7 - env: TOXENV=py37-functional - - python: 3.8 - env: TOXENV=py38 - - python: 3.8 - env: TOXENV=py38-functional - - python: 3.9 - env: TOXENV=py39 - - python: 3.9 - env: TOXENV=py39-functional diff --git a/kubernetes/base/CONTRIBUTING.md b/kubernetes/base/CONTRIBUTING.md deleted file mode 100644 index 73862f4635..0000000000 --- a/kubernetes/base/CONTRIBUTING.md +++ /dev/null @@ -1,29 +0,0 @@ -# Contributing - -Thanks for taking the time to join our community and start contributing! - -Any changes to utilities in this repo should be send as a PR to this repo. -After the PR is merged, developers should create another PR in the main repo to update the submodule. -See [this document](https://github.com/kubernetes-client/python/blob/master/devel/submodules.md) for more guidelines. - -The [Contributor Guide](https://github.com/kubernetes/community/blob/master/contributors/guide/README.md) -provides detailed instructions on how to get your ideas and bug fixes seen and accepted. - -Please remember to sign the [CNCF CLA](https://github.com/kubernetes/community/blob/master/CLA.md) and -read and observe the [Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). - -## Adding new Python modules or Python scripts -If you add a new Python module please make sure it includes the correct header -as found in: -``` -hack/boilerplate/boilerplate.py.txt -``` - -This module should not include a shebang line. - -If you add a new Python helper script intended for developers usage, it should -go into the directory `hack` and include a shebang line `#!/usr/bin/env python` -at the top in addition to rest of the boilerplate text as in all other modules. - -In addition this script's name should be added to the list -`SKIP_FILES` at the top of hack/boilerplate/boilerplate.py. diff --git a/kubernetes/base/LICENSE b/kubernetes/base/LICENSE deleted file mode 100644 index 8dada3edaf..0000000000 --- a/kubernetes/base/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/kubernetes/base/OWNERS b/kubernetes/base/OWNERS deleted file mode 100644 index 47444bf938..0000000000 --- a/kubernetes/base/OWNERS +++ /dev/null @@ -1,9 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: - - yliaog - - roycaihw -emeritus_approvers: - - mbohlool -reviewers: - - fabianvf diff --git a/kubernetes/base/README.md b/kubernetes/base/README.md deleted file mode 100644 index f916e3437a..0000000000 --- a/kubernetes/base/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# python-base - -[![Build Status](https://travis-ci.org/kubernetes-client/python-base.svg?branch=master)](https://travis-ci.org/kubernetes-client/python-base) - -This is the utility part of the [python client](https://github.com/kubernetes-client/python). It has been added to the main -repo using git submodules. This structure allow other developers to create -their own kubernetes client and still use standard kubernetes python utilities. -For more information refer to [clients-library-structure](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/csi-client-structure-proposal.md). - -## Contributing - -Please see [CONTRIBUTING.md](CONTRIBUTING.md) for instructions on how to contribute. - diff --git a/kubernetes/base/SECURITY_CONTACTS b/kubernetes/base/SECURITY_CONTACTS deleted file mode 100644 index 7992f9041a..0000000000 --- a/kubernetes/base/SECURITY_CONTACTS +++ /dev/null @@ -1,15 +0,0 @@ -# Defined below are the security contacts for this repo. -# -# They are the contact point for the Product Security Team to reach out -# to for triaging and handling of incoming issues. -# -# The below names agree to abide by the -# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) -# and will be removed and replaced if they violate that agreement. -# -# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE -# INSTRUCTIONS AT https://kubernetes.io/security/ - -mbohlool -roycaihw -yliaog diff --git a/kubernetes/base/code-of-conduct.md b/kubernetes/base/code-of-conduct.md deleted file mode 100644 index 0d15c00cf3..0000000000 --- a/kubernetes/base/code-of-conduct.md +++ /dev/null @@ -1,3 +0,0 @@ -# Kubernetes Community Code of Conduct - -Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/kubernetes/base/hack/boilerplate/boilerplate.py b/kubernetes/base/hack/boilerplate/boilerplate.py deleted file mode 100755 index e58c625144..0000000000 --- a/kubernetes/base/hack/boilerplate/boilerplate.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import datetime -import difflib -import glob -import os -import re -import sys - -# list all the files contain a shebang line and should be ignored by this -# script -SKIP_FILES = ['hack/boilerplate/boilerplate.py'] - -parser = argparse.ArgumentParser() -parser.add_argument( - "filenames", - help="list of files to check, all files if unspecified", - nargs='*') - -rootdir = os.path.dirname(__file__) + "/../../" -rootdir = os.path.abspath(rootdir) -parser.add_argument( - "--rootdir", default=rootdir, help="root directory to examine") - -default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate") -parser.add_argument( - "--boilerplate-dir", default=default_boilerplate_dir) - -parser.add_argument( - "-v", "--verbose", - help="give verbose output regarding why a file does not pass", - action="store_true") - -args = parser.parse_args() - -verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") - - -def get_refs(): - refs = {} - - for path in glob.glob(os.path.join( - args.boilerplate_dir, "boilerplate.*.txt")): - extension = os.path.basename(path).split(".")[1] - - ref_file = open(path) - ref = ref_file.read().splitlines() - ref_file.close() - refs[extension] = ref - - return refs - - -def file_passes(filename, refs, regexs): - try: - f = open(filename) - except Exception as exc: - print("Unable to open %s: %s" % (filename, exc), file=verbose_out) - return False - - data = f.read() - f.close() - - basename = os.path.basename(filename) - extension = file_extension(filename) - - if extension != "": - ref = refs[extension] - else: - ref = refs[basename] - - # remove extra content from the top of files - if extension == "sh": - p = regexs["shebang"] - (data, found) = p.subn("", data, 1) - - data = data.splitlines() - - # if our test file is smaller than the reference it surely fails! - if len(ref) > len(data): - print('File %s smaller than reference (%d < %d)' % - (filename, len(data), len(ref)), - file=verbose_out) - return False - - # trim our file to the same number of lines as the reference file - data = data[:len(ref)] - - p = regexs["year"] - for d in data: - if p.search(d): - print('File %s has the YEAR field, but missing the year of date' % - filename, file=verbose_out) - return False - - # Replace all occurrences of regex "2014|2015|2016|2017|2018" with "YEAR" - p = regexs["date"] - for i, d in enumerate(data): - (data[i], found) = p.subn('YEAR', d) - if found != 0: - break - - # if we don't match the reference at this point, fail - if ref != data: - print("Header in %s does not match reference, diff:" % - filename, file=verbose_out) - if args.verbose: - print(file=verbose_out) - for line in difflib.unified_diff( - ref, data, 'reference', filename, lineterm=''): - print(line, file=verbose_out) - print(file=verbose_out) - return False - - return True - - -def file_extension(filename): - return os.path.splitext(filename)[1].split(".")[-1].lower() - - -def normalize_files(files): - newfiles = [] - for pathname in files: - newfiles.append(pathname) - for i, pathname in enumerate(newfiles): - if not os.path.isabs(pathname): - newfiles[i] = os.path.join(args.rootdir, pathname) - - return newfiles - - -def get_files(extensions): - - files = [] - if len(args.filenames) > 0: - files = args.filenames - else: - for root, dirs, walkfiles in os.walk(args.rootdir): - for name in walkfiles: - pathname = os.path.join(root, name) - files.append(pathname) - - files = normalize_files(files) - outfiles = [] - for pathname in files: - basename = os.path.basename(pathname) - extension = file_extension(pathname) - if extension in extensions or basename in extensions: - outfiles.append(pathname) - - outfiles = list(set(outfiles) - set(normalize_files(SKIP_FILES))) - return outfiles - - -def get_dates(): - years = datetime.datetime.now().year - return '(%s)' % '|'.join(str(year) for year in range(2014, years+1)) - - -def get_regexs(): - regexs = {} - # Search for "YEAR" which exists in the boilerplate, - # but shouldn't in the real thing - regexs["year"] = re.compile('YEAR') - # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year - # as a regex like: "(2014|2015|2016|2017|2018)"; - # company holder names can be anything - regexs["date"] = re.compile(get_dates()) - # strip #!.* from shell scripts - regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) - return regexs - - -def main(): - regexs = get_regexs() - refs = get_refs() - filenames = get_files(refs.keys()) - - for filename in filenames: - if not file_passes(filename, refs, regexs): - print(filename, file=sys.stdout) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/kubernetes/base/hack/boilerplate/boilerplate.py.txt b/kubernetes/base/hack/boilerplate/boilerplate.py.txt deleted file mode 100644 index 34cb349c40..0000000000 --- a/kubernetes/base/hack/boilerplate/boilerplate.py.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright YEAR The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/kubernetes/base/hack/boilerplate/boilerplate.sh.txt b/kubernetes/base/hack/boilerplate/boilerplate.sh.txt deleted file mode 100644 index 34cb349c40..0000000000 --- a/kubernetes/base/hack/boilerplate/boilerplate.sh.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright YEAR The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/kubernetes/base/run_tox.sh b/kubernetes/base/run_tox.sh deleted file mode 100755 index fe5b48c903..0000000000 --- a/kubernetes/base/run_tox.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -RUNNING_DIR=$(pwd) -TMP_DIR=$(mktemp -d) - -function cleanup() -{ - cd "${RUNNING_DIR}" -} -trap cleanup EXIT SIGINT - - -SCRIPT_ROOT=$(dirname "${BASH_SOURCE}") -pushd "${SCRIPT_ROOT}" > /dev/null -SCRIPT_ROOT=`pwd` -popd > /dev/null - -cd "${TMP_DIR}" -git clone https://github.com/kubernetes-client/python.git -cd python -git config user.email "kubernetes-client@k8s.com" -git config user.name "kubernetes client" -git rm -rf kubernetes/base -git commit -m "DO NOT MERGE, removing submodule for testing only" -mkdir kubernetes/base -cp -r "${SCRIPT_ROOT}/." kubernetes/base -rm -rf kubernetes/base/.git -rm -rf kubernetes/base/.tox -git add kubernetes/base -git commit -m "DO NOT MERGE, adding changes for testing." -git status - -echo "Running tox from the main repo on $TOXENV environment" -# Run the user-provided command. -"${@}" diff --git a/kubernetes/base/tox.ini b/kubernetes/base/tox.ini deleted file mode 100644 index 91cbf9b555..0000000000 --- a/kubernetes/base/tox.ini +++ /dev/null @@ -1,21 +0,0 @@ -[tox] -skipsdist = True -envlist = - py3{8,9} - py31{0,1,2,3,4,5} - py3{8,9}-functional - py31{0,1,2,3,4,5}-functional - -[testenv] -passenv = - TOXENV - CI - TRAVIS - TRAVIS_* -deps = - pytest -commands = - python -V - pytest - ./run_tox.sh pytest - diff --git a/kubernetes/config b/kubernetes/config deleted file mode 120000 index e65cfe84e5..0000000000 --- a/kubernetes/config +++ /dev/null @@ -1 +0,0 @@ -base/config \ No newline at end of file diff --git a/kubernetes/base/config/__init__.py b/kubernetes/config/__init__.py similarity index 100% rename from kubernetes/base/config/__init__.py rename to kubernetes/config/__init__.py diff --git a/kubernetes/base/config/config_exception.py b/kubernetes/config/config_exception.py similarity index 100% rename from kubernetes/base/config/config_exception.py rename to kubernetes/config/config_exception.py diff --git a/kubernetes/base/config/dateutil.py b/kubernetes/config/dateutil.py similarity index 100% rename from kubernetes/base/config/dateutil.py rename to kubernetes/config/dateutil.py diff --git a/kubernetes/base/config/dateutil_test.py b/kubernetes/config/dateutil_test.py similarity index 100% rename from kubernetes/base/config/dateutil_test.py rename to kubernetes/config/dateutil_test.py diff --git a/kubernetes/base/config/exec_provider.py b/kubernetes/config/exec_provider.py similarity index 100% rename from kubernetes/base/config/exec_provider.py rename to kubernetes/config/exec_provider.py diff --git a/kubernetes/base/config/exec_provider_test.py b/kubernetes/config/exec_provider_test.py similarity index 100% rename from kubernetes/base/config/exec_provider_test.py rename to kubernetes/config/exec_provider_test.py diff --git a/kubernetes/base/config/incluster_config.py b/kubernetes/config/incluster_config.py similarity index 100% rename from kubernetes/base/config/incluster_config.py rename to kubernetes/config/incluster_config.py diff --git a/kubernetes/base/config/incluster_config_test.py b/kubernetes/config/incluster_config_test.py similarity index 100% rename from kubernetes/base/config/incluster_config_test.py rename to kubernetes/config/incluster_config_test.py diff --git a/kubernetes/base/config/kube_config.py b/kubernetes/config/kube_config.py similarity index 100% rename from kubernetes/base/config/kube_config.py rename to kubernetes/config/kube_config.py diff --git a/kubernetes/base/config/kube_config_test.py b/kubernetes/config/kube_config_test.py similarity index 100% rename from kubernetes/base/config/kube_config_test.py rename to kubernetes/config/kube_config_test.py diff --git a/kubernetes/dynamic b/kubernetes/dynamic deleted file mode 120000 index e896b54ffd..0000000000 --- a/kubernetes/dynamic +++ /dev/null @@ -1 +0,0 @@ -base/dynamic \ No newline at end of file diff --git a/kubernetes/base/dynamic/__init__.py b/kubernetes/dynamic/__init__.py similarity index 100% rename from kubernetes/base/dynamic/__init__.py rename to kubernetes/dynamic/__init__.py diff --git a/kubernetes/base/dynamic/client.py b/kubernetes/dynamic/client.py similarity index 100% rename from kubernetes/base/dynamic/client.py rename to kubernetes/dynamic/client.py diff --git a/kubernetes/base/dynamic/client_test.py b/kubernetes/dynamic/client_test.py similarity index 100% rename from kubernetes/base/dynamic/client_test.py rename to kubernetes/dynamic/client_test.py diff --git a/kubernetes/base/dynamic/discovery.py b/kubernetes/dynamic/discovery.py similarity index 100% rename from kubernetes/base/dynamic/discovery.py rename to kubernetes/dynamic/discovery.py diff --git a/kubernetes/base/dynamic/exceptions.py b/kubernetes/dynamic/exceptions.py similarity index 100% rename from kubernetes/base/dynamic/exceptions.py rename to kubernetes/dynamic/exceptions.py diff --git a/kubernetes/base/dynamic/resource.py b/kubernetes/dynamic/resource.py similarity index 100% rename from kubernetes/base/dynamic/resource.py rename to kubernetes/dynamic/resource.py diff --git a/kubernetes/base/dynamic/test_client.py b/kubernetes/dynamic/test_client.py similarity index 100% rename from kubernetes/base/dynamic/test_client.py rename to kubernetes/dynamic/test_client.py diff --git a/kubernetes/base/dynamic/test_discovery.py b/kubernetes/dynamic/test_discovery.py similarity index 100% rename from kubernetes/base/dynamic/test_discovery.py rename to kubernetes/dynamic/test_discovery.py diff --git a/kubernetes/leaderelection b/kubernetes/leaderelection deleted file mode 120000 index 30e0567f73..0000000000 --- a/kubernetes/leaderelection +++ /dev/null @@ -1 +0,0 @@ -base/leaderelection \ No newline at end of file diff --git a/kubernetes/base/leaderelection/README.md b/kubernetes/leaderelection/README.md similarity index 100% rename from kubernetes/base/leaderelection/README.md rename to kubernetes/leaderelection/README.md diff --git a/kubernetes/base/leaderelection/__init__.py b/kubernetes/leaderelection/__init__.py similarity index 100% rename from kubernetes/base/leaderelection/__init__.py rename to kubernetes/leaderelection/__init__.py diff --git a/kubernetes/base/leaderelection/electionconfig.py b/kubernetes/leaderelection/electionconfig.py similarity index 100% rename from kubernetes/base/leaderelection/electionconfig.py rename to kubernetes/leaderelection/electionconfig.py diff --git a/kubernetes/base/leaderelection/example.py b/kubernetes/leaderelection/example.py similarity index 100% rename from kubernetes/base/leaderelection/example.py rename to kubernetes/leaderelection/example.py diff --git a/kubernetes/base/leaderelection/leaderelection.py b/kubernetes/leaderelection/leaderelection.py similarity index 100% rename from kubernetes/base/leaderelection/leaderelection.py rename to kubernetes/leaderelection/leaderelection.py diff --git a/kubernetes/base/leaderelection/leaderelection_test.py b/kubernetes/leaderelection/leaderelection_test.py similarity index 100% rename from kubernetes/base/leaderelection/leaderelection_test.py rename to kubernetes/leaderelection/leaderelection_test.py diff --git a/kubernetes/base/leaderelection/leaderelectionrecord.py b/kubernetes/leaderelection/leaderelectionrecord.py similarity index 100% rename from kubernetes/base/leaderelection/leaderelectionrecord.py rename to kubernetes/leaderelection/leaderelectionrecord.py diff --git a/kubernetes/base/leaderelection/resourcelock/__init__.py b/kubernetes/leaderelection/resourcelock/__init__.py similarity index 100% rename from kubernetes/base/leaderelection/resourcelock/__init__.py rename to kubernetes/leaderelection/resourcelock/__init__.py diff --git a/kubernetes/base/leaderelection/resourcelock/configmaplock.py b/kubernetes/leaderelection/resourcelock/configmaplock.py similarity index 100% rename from kubernetes/base/leaderelection/resourcelock/configmaplock.py rename to kubernetes/leaderelection/resourcelock/configmaplock.py diff --git a/kubernetes/base/leaderelection/resourcelock/leaselock.py b/kubernetes/leaderelection/resourcelock/leaselock.py similarity index 100% rename from kubernetes/base/leaderelection/resourcelock/leaselock.py rename to kubernetes/leaderelection/resourcelock/leaselock.py diff --git a/kubernetes/base/leaderelection/resourcelock/leaselock_test.py b/kubernetes/leaderelection/resourcelock/leaselock_test.py similarity index 100% rename from kubernetes/base/leaderelection/resourcelock/leaselock_test.py rename to kubernetes/leaderelection/resourcelock/leaselock_test.py diff --git a/kubernetes/stream b/kubernetes/stream deleted file mode 120000 index 387e18fe54..0000000000 --- a/kubernetes/stream +++ /dev/null @@ -1 +0,0 @@ -base/stream \ No newline at end of file diff --git a/kubernetes/base/stream/__init__.py b/kubernetes/stream/__init__.py similarity index 100% rename from kubernetes/base/stream/__init__.py rename to kubernetes/stream/__init__.py diff --git a/kubernetes/base/stream/stream.py b/kubernetes/stream/stream.py similarity index 100% rename from kubernetes/base/stream/stream.py rename to kubernetes/stream/stream.py diff --git a/kubernetes/base/stream/stream_test.py b/kubernetes/stream/stream_test.py similarity index 100% rename from kubernetes/base/stream/stream_test.py rename to kubernetes/stream/stream_test.py diff --git a/kubernetes/base/stream/ws_client.py b/kubernetes/stream/ws_client.py similarity index 100% rename from kubernetes/base/stream/ws_client.py rename to kubernetes/stream/ws_client.py diff --git a/kubernetes/base/stream/ws_client_test.py b/kubernetes/stream/ws_client_test.py similarity index 100% rename from kubernetes/base/stream/ws_client_test.py rename to kubernetes/stream/ws_client_test.py diff --git a/kubernetes/watch b/kubernetes/watch deleted file mode 120000 index b3079b32f6..0000000000 --- a/kubernetes/watch +++ /dev/null @@ -1 +0,0 @@ -base/watch \ No newline at end of file diff --git a/kubernetes/base/watch/__init__.py b/kubernetes/watch/__init__.py similarity index 100% rename from kubernetes/base/watch/__init__.py rename to kubernetes/watch/__init__.py diff --git a/kubernetes/base/watch/watch.py b/kubernetes/watch/watch.py similarity index 100% rename from kubernetes/base/watch/watch.py rename to kubernetes/watch/watch.py diff --git a/kubernetes/base/watch/watch_test.py b/kubernetes/watch/watch_test.py similarity index 100% rename from kubernetes/base/watch/watch_test.py rename to kubernetes/watch/watch_test.py