feat: Make use of status conditions when activating maintenance mode (#1145)

Related to: https://github.com/harvester/harvester/issues/9022

Signed-off-by: Volker Theile <vtheile@suse.com>

pkg/harvester/config/feature-flags.js

@@ -81,7 +81,10 @@ const FEATURE_FLAGS = {
     'lvmDmThinOnly',
     'highPerformanceStorage',
   ],
-  'v1.10.0': [],
+  'v1.10.0': [
+    'maintenanceModeConditions',
+    'maintenanceModeDrainTimeoutSetting',
+  ],
 };
 
 const generateFeatureFlags = () => {

pkg/harvester/config/labels-annotations.js

@@ -22,7 +22,7 @@ export const HCI = {
   VM_NAME_PREFIX:                   'harvesterhci.io/vmNamePrefix',
   VM_DISPLAY_NAME:                  'harvesterhci.io/vmDisplayName',
   VM_RESERVED_MEMORY:               'harvesterhci.io/reservedMemory',
-  MAINTENANCE_STATUS:               'harvesterhci.io/maintain-status',
+  MAINTENANCE_STATUS:               'harvesterhci.io/maintain-status', // Deprecated >= v1.10.0
   HOST_CUSTOM_NAME:                 'harvesterhci.io/host-custom-name',
   HOST_CONSOLE_URL:                 'harvesterhci.io/host-console-url',
   RESTORE_NAME:                     'restore.harvesterhci.io/name',

pkg/harvester/config/settings.ts

@@ -45,7 +45,8 @@ export const HCI_SETTING = {
   MAX_HOTPLUG_RATIO:                        'max-hotplug-ratio',
   KUBEVIRT_MIGRATION:                       'kubevirt-migration',
   INSTANCE_MANAGER_RESOURCES:               'instance-manager-resources',
-  CLUSTER_POD_SECURITY_STANDARD:            'cluster-pod-security-standard'
+  CLUSTER_POD_SECURITY_STANDARD:            'cluster-pod-security-standard',
+  MAINTENANCE_MODE_DRAIN_TIMEOUT:           'maintenance-mode-drain-timeout'
 };
 
 export const HCI_ALLOWED_SETTINGS = {
@@ -148,6 +149,9 @@ export const HCI_ALLOWED_SETTINGS = {
   },
   [HCI_SETTING.CLUSTER_POD_SECURITY_STANDARD]: {
     kind: 'json', from: 'import', canReset: true, featureFlag: 'clusterPodSecurityStandardSetting'
+  },
+  [HCI_SETTING.MAINTENANCE_MODE_DRAIN_TIMEOUT]: {
+    kind: 'number', canReset: true, featureFlag: 'maintenanceModeDrainTimeoutSetting'
   }
 };
 

pkg/harvester/l10n/en-us.yaml

@@ -286,6 +286,7 @@ harvester:
     createTemplate: Generate Template
     enableMaintenance: Enable Maintenance Mode
     disableMaintenance: Disable Maintenance Mode
+    clearMaintenance: Clear Maintenance Mode Error
     enableCPUManager: Enable CPU Manager
     disableCPUManager: Disable CPU Manager
     cordon: Cordon
@@ -2541,6 +2542,7 @@ advancedSettings:
     'harv-kubevirt-migration': 'Configure cluster-wide KubeVirt live migration parameters.'
     'harv-instance-manager-resources': 'Configure resource percentage reservations for Longhorn instance manager V1 and V2. Valid instance manager CPU range between 0 - 40.'
     'harv-cluster-pod-security-standard': 'Enforce Kubernetes Pod Security Standards (PSS) at the cluster level.'
+    'harv-maintenance-mode-drain-timeout': 'Maintenance mode drain timeout in minutes. Use 0 to disable the timeout.'
 
 typeLabel:
   kubevirt.io.virtualmachine: |-

pkg/harvester/models/harvester/node.js

@@ -23,6 +23,13 @@ const ALLOW_SYSTEM_LABEL_KEYS = [
 
 const HEALTHY = 'healthy';
 const WARNING = 'warning';
+const MAINTENANCE_MODE = 'MaintenanceMode';
+const MAINTENANCE_ERROR = 'Error';
+const MAINTENANCE_ENTERING_REASONS = ['Validating', 'Draining', 'Evacuating'];
+const MAINTENANCE_COMPLETED = 'Completed';
+// Legacy `harvesterhci.io/maintain-status` annotation values (clusters < v1.10)
+const LEGACY_MAINTENANCE_RUNNING = 'running';
+const LEGACY_MAINTENANCE_COMPLETED = 'completed';
 
 export default class HciNode extends HarvesterResource {
   get _availableActions() {
@@ -58,6 +65,14 @@ export default class HciNode extends HarvesterResource {
       total:   1
     };
 
+    const clearMaintenance = {
+      action:  'clearMaintenanceMode',
+      enabled: this.hasAction('clearMaintenanceMode'),
+      icon:    'icon icon-fw icon-close',
+      label:   this.t('harvester.action.clearMaintenance'),
+      total:   1
+    };
+
     const enableCPUManager = {
       action:  'enableCPUManager',
       enabled: this.cpuPinningFeatureEnabled && this.hasAction('enableCPUManager') && !this.isCPUManagerEnableInProgress && !this.isCPUManagerEnabled && !this.isEtcd, // witness node doesn't have CPU manager
@@ -103,6 +118,7 @@ export default class HciNode extends HarvesterResource {
       uncordon,
       enableMaintenance,
       disableMaintenance,
+      clearMaintenance,
       enableCPUManager,
       disableCPUManager,
       shutDown,
@@ -198,6 +214,10 @@ export default class HciNode extends HarvesterResource {
   }
 
   get stateDisplay() {
+    if (this.isMaintenanceModeError) {
+      return super.stateDisplay;
+    }
+
     if (this.isEnteringMaintenance) {
       return 'Entering maintenance mode';
     }
@@ -246,6 +266,10 @@ export default class HciNode extends HarvesterResource {
   }
 
   get stateDescription() {
+    if (this.isMaintenanceModeError) {
+      return super.stateDescription;
+    }
+
     const currentIP = this.metadata?.annotations?.[HCI_ANNOTATIONS.CURRENT_IP];
     const initIP = this.metadata?.annotations?.[HCI_ANNOTATIONS.INIT_IP];
 
@@ -325,7 +349,11 @@ export default class HciNode extends HarvesterResource {
   }
 
   disableMaintenanceMode() {
-    this.doAction('disableMaintenanceMode', {});
+    this.doActionGrowl('disableMaintenanceMode', {});
+  }
+
+  clearMaintenanceMode() {
+    this.doActionGrowl('clearMaintenanceMode', {});
   }
 
   enableCPUManager() {
@@ -343,11 +371,32 @@ export default class HciNode extends HarvesterResource {
     );
   }
 
+  get maintenanceModeCondition() {
+    return (this.status?.conditions || []).find(
+      (c) => c.type === MAINTENANCE_MODE
+    );
+  }
+
+  get maintenanceModeConditionsEnabled() {
+    return this.$rootGetters['harvester-common/getFeatureEnabled']('maintenanceModeConditions');
+  }
+
+  // Clusters < v1.10 only expose the `harvesterhci.io/maintain-status` annotation
+  get legacyMaintenanceStatus() {
+    return this.metadata?.annotations?.[HCI_ANNOTATIONS.MAINTENANCE_STATUS];
+  }
+
+  get isMaintenanceModeError() {
+    return this.maintenanceModeConditionsEnabled && this.maintenanceModeCondition?.reason === MAINTENANCE_ERROR;
+  }
+
   get isMigratable() {
     const states = ['in-progress', 'unavailable'];
 
+    const engaged = this.maintenanceModeConditionsEnabled ? this.maintenanceModeCondition?.status === 'True' : !!this.legacyMaintenanceStatus;
+
     return (
-      !this.metadata?.annotations?.[HCI_ANNOTATIONS.MAINTENANCE_STATUS] &&
+      !engaged &&
       !this.isUnSchedulable &&
       !states.includes(this.state)
     );
@@ -362,17 +411,26 @@ export default class HciNode extends HarvesterResource {
   }
 
   get isEnteringMaintenance() {
+    if (!this.maintenanceModeConditionsEnabled) {
+      return this.legacyMaintenanceStatus === LEGACY_MAINTENANCE_RUNNING;
+    }
+
+    const cond = this.maintenanceModeCondition;
+
     return (
-      this.metadata?.annotations?.[HCI_ANNOTATIONS.MAINTENANCE_STATUS] ===
+      cond?.status === 'True' &&
-      'running'
+      MAINTENANCE_ENTERING_REASONS.includes(cond?.reason)
     );
   }
 
   get isMaintenance() {
-    return (
+    if (!this.maintenanceModeConditionsEnabled) {
-      this.metadata?.annotations?.[HCI_ANNOTATIONS.MAINTENANCE_STATUS] ===
+      return this.legacyMaintenanceStatus === LEGACY_MAINTENANCE_COMPLETED;
-      'completed'
+    }
-    );
+
+    const cond = this.maintenanceModeCondition;
+
+    return cond?.status === 'True' && cond?.reason === MAINTENANCE_COMPLETED;
   }
 
   get cpuPinningFeatureEnabled() {