001/* 002 * VM-Operator 003 * Copyright (C) 2023 Michael N. Lipp 004 * 005 * This program is free software: you can redistribute it and/or modify 006 * it under the terms of the GNU Affero General Public License as 007 * published by the Free Software Foundation, either version 3 of the 008 * License, or (at your option) any later version. 009 * 010 * This program is distributed in the hope that it will be useful, 011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 013 * GNU Affero General Public License for more details. 014 * 015 * You should have received a copy of the GNU Affero General Public License 016 * along with this program. If not, see <https://www.gnu.org/licenses/>. 017 */ 018 019package org.jdrupes.vmoperator.manager; 020 021import com.fasterxml.jackson.core.JsonProcessingException; 022import com.fasterxml.jackson.databind.ObjectMapper; 023import freemarker.template.AdapterTemplateModel; 024import freemarker.template.Configuration; 025import freemarker.template.DefaultObjectWrapperBuilder; 026import freemarker.template.SimpleNumber; 027import freemarker.template.SimpleScalar; 028import freemarker.template.TemplateException; 029import freemarker.template.TemplateExceptionHandler; 030import freemarker.template.TemplateHashModel; 031import freemarker.template.TemplateMethodModelEx; 032import freemarker.template.TemplateModelException; 033import io.kubernetes.client.custom.Quantity; 034import io.kubernetes.client.openapi.ApiException; 035import io.kubernetes.client.openapi.models.V1ObjectMeta; 036import io.kubernetes.client.util.generic.options.ListOptions; 037import java.io.IOException; 038import java.math.BigDecimal; 039import java.math.BigInteger; 040import java.net.URI; 041import java.net.URISyntaxException; 042import java.util.HashMap; 043import java.util.List; 044import java.util.Map; 045import java.util.Optional; 046import static org.jdrupes.vmoperator.common.Constants.APP_NAME; 047import org.jdrupes.vmoperator.common.Convertions; 048import org.jdrupes.vmoperator.common.K8sClient; 049import org.jdrupes.vmoperator.common.K8sObserver; 050import org.jdrupes.vmoperator.common.K8sV1SecretStub; 051import org.jdrupes.vmoperator.common.VmDefinition; 052import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET; 053import org.jdrupes.vmoperator.manager.events.ResetVm; 054import org.jdrupes.vmoperator.manager.events.VmChannel; 055import org.jdrupes.vmoperator.manager.events.VmDefChanged; 056import org.jdrupes.vmoperator.util.DataPath; 057import org.jdrupes.vmoperator.util.ExtendedObjectWrapper; 058import org.jgrapes.core.Channel; 059import org.jgrapes.core.Component; 060import org.jgrapes.core.annotation.Handler; 061import org.jgrapes.util.events.ConfigurationUpdate; 062 063/** 064 * Adapts Kubenetes resources for instances of the Runner 065 * application (the VMs) to changes in VM definitions (the CRs). 066 * 067 * In particular, the reconciler generates and updates: 068 * 069 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) 070 * for storage used by all VMs as a common repository for CDROM images. 071 * 072 * * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/) 073 * that defines the configuration file for the runner. 074 * 075 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) 076 * for 1 MiB of persistent storage used by the Runner (referred to as the 077 * "runnerDataPvc") 078 * 079 * * The PVCs for the VM's disks. 080 * 081 * * A [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) with the 082 * runner instance[^oldSts]. 083 * 084 * * (Optional) A load balancer 085 * [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) 086 * that allows the user to access a VM's console without knowing which 087 * node it runs on. 088 * 089 * [^oldSts]: Before version 3.4, the operator created a 090 * [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) 091 * that created the pod. 092 * 093 * The reconciler is part of the {@link Controller} component. It's 094 * configuration properties are therefore defined in 095 * ```yaml 096 * "/Manager": 097 * "/Controller": 098 * "/Reconciler": 099 * ... 100 * ``` 101 * 102 * The reconciler supports the following configuration properties: 103 * 104 * * `runnerDataPvc.storageClassName`: The storage class name 105 * to be used for the "runnerDataPvc" (the small volume used 106 * by the runner for information such as the EFI variables). By 107 * default, no `storageClassName` is generated, which causes 108 * Kubernetes to use storage from the default storage class. 109 * Define this if you want to use a specific storage class. 110 * 111 * * `cpuOvercommit`: The amount by which the current cpu count 112 * from the VM definition is divided when generating the 113 * [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) 114 * properties for the VM (defaults to 2). 115 * 116 * * `ramOvercommit`: The amount by which the current ram size 117 * from the VM definition is divided when generating the 118 * [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) 119 * properties for the VM (defaults to 1.25). 120 * 121 * * `loadBalancerService`: If defined, causes a load balancer service 122 * to be created. This property may be a boolean or 123 * YAML that defines additional labels or annotations to be merged 124 * into the service defintion. Here's an example for using 125 * [MetalLb](https://metallb.universe.tf/) as "internal load balancer": 126 * ```yaml 127 * loadBalancerService: 128 * annotations: 129 * metallb.universe.tf/loadBalancerIPs: 192.168.168.1 130 * metallb.universe.tf/ip-allocated-from-pool: single-common 131 * metallb.universe.tf/allow-shared-ip: single-common 132 * ``` 133 * This makes all VM consoles available at IP address 192.168.168.1 134 * with the port numbers from the VM definitions. 135 * 136 * * `loggingProperties`: If defined, specifies the default logging 137 * properties to be used by the runners managed by the controller. 138 * This property is a string that holds the content of 139 * a logging.properties file. 140 */ 141@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis", 142 "PMD.AvoidDuplicateLiterals" }) 143public class Reconciler extends Component { 144 145 /** The Constant mapper. */ 146 @SuppressWarnings("PMD.FieldNamingConventions") 147 protected static final ObjectMapper mapper = new ObjectMapper(); 148 149 @SuppressWarnings("PMD.SingularField") 150 private final Configuration fmConfig; 151 private final ConfigMapReconciler cmReconciler; 152 private final DisplaySecretReconciler dsReconciler; 153 private final StatefulSetReconciler stsReconciler; 154 private final PvcReconciler pvcReconciler; 155 private final PodReconciler podReconciler; 156 private final LoadBalancerReconciler lbReconciler; 157 @SuppressWarnings("PMD.UseConcurrentHashMap") 158 private final Map<String, Object> config = new HashMap<>(); 159 160 /** 161 * Instantiates a new reconciler. 162 * 163 * @param componentChannel the component channel 164 */ 165 public Reconciler(Channel componentChannel) { 166 super(componentChannel); 167 168 // Configure freemarker library 169 fmConfig = new Configuration(Configuration.VERSION_2_3_32); 170 fmConfig.setDefaultEncoding("utf-8"); 171 fmConfig.setObjectWrapper(new ExtendedObjectWrapper( 172 fmConfig.getIncompatibleImprovements())); 173 fmConfig.setTemplateExceptionHandler( 174 TemplateExceptionHandler.RETHROW_HANDLER); 175 fmConfig.setLogTemplateExceptions(false); 176 fmConfig.setClassForTemplateLoading(Reconciler.class, ""); 177 178 cmReconciler = new ConfigMapReconciler(fmConfig); 179 dsReconciler = new DisplaySecretReconciler(); 180 stsReconciler = new StatefulSetReconciler(fmConfig); 181 pvcReconciler = new PvcReconciler(fmConfig); 182 podReconciler = new PodReconciler(fmConfig); 183 lbReconciler = new LoadBalancerReconciler(fmConfig); 184 } 185 186 /** 187 * Configures the component. 188 * 189 * @param event the event 190 */ 191 @Handler 192 public void onConfigurationUpdate(ConfigurationUpdate event) { 193 event.structured(componentPath()).ifPresent(c -> { 194 config.putAll(c); 195 }); 196 } 197 198 /** 199 * Handles the change event. 200 * 201 * @param event the event 202 * @param channel the channel 203 * @throws ApiException the api exception 204 * @throws TemplateException the template exception 205 * @throws IOException Signals that an I/O exception has occurred. 206 */ 207 @Handler 208 @SuppressWarnings("PMD.ConfusingTernary") 209 public void onVmDefChanged(VmDefChanged event, VmChannel channel) 210 throws ApiException, TemplateException, IOException { 211 // We're only interested in "spec" changes. 212 if (!event.specChanged()) { 213 return; 214 } 215 216 // Ownership relationships takes care of deletions 217 if (event.type() == K8sObserver.ResponseType.DELETED) { 218 logger.fine( 219 () -> "VM \"" + event.vmDefinition().name() + "\" deleted"); 220 return; 221 } 222 223 // Create model for processing templates 224 Map<String, Object> model 225 = prepareModel(channel.client(), event.vmDefinition()); 226 var configMap = cmReconciler.reconcile(model, channel); 227 model.put("cm", configMap); 228 dsReconciler.reconcile(event, model, channel); 229 // Manage (eventual) removal of stateful set. 230 stsReconciler.reconcile(event, model, channel); 231 pvcReconciler.reconcile(event, model, channel); 232 podReconciler.reconcile(event, model, channel); 233 lbReconciler.reconcile(event, model, channel); 234 } 235 236 /** 237 * Reset the VM by incrementing the reset count and doing a 238 * partial reconcile (configmap only). 239 * 240 * @param event the event 241 * @param channel the channel 242 * @throws IOException 243 * @throws ApiException 244 * @throws TemplateException 245 */ 246 @Handler 247 public void onResetVm(ResetVm event, VmChannel channel) 248 throws ApiException, IOException, TemplateException { 249 var vmDef = channel.vmDefinition(); 250 vmDef.extra("resetCount", vmDef.<Long> extra("resetCount") + 1); 251 Map<String, Object> model 252 = prepareModel(channel.client(), channel.vmDefinition()); 253 cmReconciler.reconcile(model, channel); 254 } 255 256 @SuppressWarnings({ "PMD.CognitiveComplexity", "PMD.NPathComplexity" }) 257 private Map<String, Object> prepareModel(K8sClient client, 258 VmDefinition vmDef) throws TemplateModelException, ApiException { 259 @SuppressWarnings("PMD.UseConcurrentHashMap") 260 Map<String, Object> model = new HashMap<>(); 261 model.put("managerVersion", 262 Optional.ofNullable(Reconciler.class.getPackage() 263 .getImplementationVersion()).orElse("(Unknown)")); 264 model.put("cr", vmDef); 265 model.put("constants", 266 (TemplateHashModel) new DefaultObjectWrapperBuilder( 267 Configuration.VERSION_2_3_32) 268 .build().getStaticModels() 269 .get(Constants.class.getName())); 270 model.put("reconciler", config); 271 272 // Check if we have a display secret 273 ListOptions options = new ListOptions(); 274 options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + "," 275 + "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + "," 276 + "app.kubernetes.io/instance=" + vmDef.name()); 277 var dsStub = K8sV1SecretStub 278 .list(client, vmDef.namespace(), options) 279 .stream() 280 .findFirst(); 281 if (dsStub.isPresent()) { 282 dsStub.get().model().ifPresent(m -> { 283 model.put("displaySecret", m.getMetadata().getName()); 284 }); 285 } 286 287 // Methods 288 model.put("parseQuantity", parseQuantityModel); 289 model.put("formatMemory", formatMemoryModel); 290 model.put("imageLocation", imgageLocationModel); 291 model.put("adjustCloudInitMeta", adjustCloudInitMetaModel); 292 model.put("toJson", toJsonModel); 293 return model; 294 } 295 296 private final TemplateMethodModelEx parseQuantityModel 297 = new TemplateMethodModelEx() { 298 @Override 299 @SuppressWarnings("PMD.PreserveStackTrace") 300 public Object exec(@SuppressWarnings("rawtypes") List arguments) 301 throws TemplateModelException { 302 var arg = arguments.get(0); 303 if (arg instanceof SimpleNumber number) { 304 return number.getAsNumber(); 305 } 306 try { 307 return Quantity.fromString(arg.toString()).getNumber(); 308 } catch (NumberFormatException e) { 309 throw new TemplateModelException("Cannot parse memory " 310 + "specified as \"" + arg + "\": " + e.getMessage()); 311 } 312 } 313 }; 314 315 private final TemplateMethodModelEx formatMemoryModel 316 = new TemplateMethodModelEx() { 317 @Override 318 @SuppressWarnings("PMD.PreserveStackTrace") 319 public Object exec(@SuppressWarnings("rawtypes") List arguments) 320 throws TemplateModelException { 321 var arg = arguments.get(0); 322 if (arg instanceof SimpleNumber number) { 323 arg = number.getAsNumber(); 324 } 325 BigInteger bigInt; 326 if (arg instanceof BigInteger value) { 327 bigInt = value; 328 } else if (arg instanceof BigDecimal dec) { 329 try { 330 bigInt = dec.toBigIntegerExact(); 331 } catch (ArithmeticException e) { 332 return arg; 333 } 334 } else if (arg instanceof Integer value) { 335 bigInt = BigInteger.valueOf(value); 336 } else if (arg instanceof Long value) { 337 bigInt = BigInteger.valueOf(value); 338 } else { 339 return arg; 340 } 341 return Convertions.formatMemory(bigInt); 342 } 343 }; 344 345 private final TemplateMethodModelEx imgageLocationModel 346 = new TemplateMethodModelEx() { 347 @Override 348 @SuppressWarnings({ "PMD.PreserveStackTrace", 349 "PMD.AvoidLiteralsInIfCondition" }) 350 public Object exec(@SuppressWarnings("rawtypes") List arguments) 351 throws TemplateModelException { 352 var image = ((SimpleScalar) arguments.get(0)).getAsString(); 353 if (image.isEmpty()) { 354 return ""; 355 } 356 try { 357 var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH 358 + "/").resolve(image); 359 if ("file".equals(imageUri.getScheme())) { 360 return imageUri.getPath(); 361 } 362 return imageUri.toString(); 363 } catch (URISyntaxException e) { 364 logger.warning(() -> "Invalid CDROM image: " + image); 365 } 366 return image; 367 } 368 }; 369 370 private final TemplateMethodModelEx adjustCloudInitMetaModel 371 = new TemplateMethodModelEx() { 372 @Override 373 @SuppressWarnings("PMD.PreserveStackTrace") 374 public Object exec(@SuppressWarnings("rawtypes") List arguments) 375 throws TemplateModelException { 376 @SuppressWarnings("unchecked") 377 var res = (Map<String, Object>) DataPath 378 .deepCopy(((AdapterTemplateModel) arguments.get(0)) 379 .getAdaptedObject(Object.class)); 380 var metadata 381 = (V1ObjectMeta) ((AdapterTemplateModel) arguments.get(1)) 382 .getAdaptedObject(Object.class); 383 if (!res.containsKey("instance-id")) { 384 res.put("instance-id", 385 Optional.ofNullable(metadata.getResourceVersion()) 386 .map(s -> "v" + s).orElse("v1")); 387 } 388 if (!res.containsKey("local-hostname")) { 389 res.put("local-hostname", metadata.getName()); 390 } 391 return res; 392 } 393 }; 394 395 private final TemplateMethodModelEx toJsonModel 396 = new TemplateMethodModelEx() { 397 @Override 398 @SuppressWarnings("PMD.PreserveStackTrace") 399 public Object exec(@SuppressWarnings("rawtypes") List arguments) 400 throws TemplateModelException { 401 try { 402 return mapper.writeValueAsString( 403 ((AdapterTemplateModel) arguments.get(0)) 404 .getAdaptedObject(Object.class)); 405 } catch (JsonProcessingException e) { 406 return "{}"; 407 } 408 } 409 }; 410}