001/* 002 * VM-Operator 003 * Copyright (C) 2023 Michael N. Lipp 004 * 005 * This program is free software: you can redistribute it and/or modify 006 * it under the terms of the GNU Affero General Public License as 007 * published by the Free Software Foundation, either version 3 of the 008 * License, or (at your option) any later version. 009 * 010 * This program is distributed in the hope that it will be useful, 011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 013 * GNU Affero General Public License for more details. 014 * 015 * You should have received a copy of the GNU Affero General Public License 016 * along with this program. If not, see <https://www.gnu.org/licenses/>. 017 */ 018 019package org.jdrupes.vmoperator.manager; 020 021import com.google.gson.JsonArray; 022import com.google.gson.JsonObject; 023import freemarker.template.Configuration; 024import freemarker.template.DefaultObjectWrapperBuilder; 025import freemarker.template.SimpleNumber; 026import freemarker.template.TemplateException; 027import freemarker.template.TemplateExceptionHandler; 028import freemarker.template.TemplateHashModel; 029import freemarker.template.TemplateMethodModelEx; 030import freemarker.template.TemplateModelException; 031import io.kubernetes.client.custom.Quantity; 032import io.kubernetes.client.openapi.ApiException; 033import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject; 034import io.kubernetes.client.util.generic.options.ListOptions; 035import java.io.IOException; 036import java.math.BigDecimal; 037import java.math.BigInteger; 038import java.net.URI; 039import java.net.URISyntaxException; 040import java.util.HashMap; 041import java.util.List; 042import java.util.Map; 043import java.util.Optional; 044import static org.jdrupes.vmoperator.common.Constants.APP_NAME; 045import org.jdrupes.vmoperator.common.Convertions; 046import org.jdrupes.vmoperator.common.K8sClient; 047import org.jdrupes.vmoperator.common.K8sDynamicModel; 048import org.jdrupes.vmoperator.common.K8sObserver; 049import org.jdrupes.vmoperator.common.K8sV1SecretStub; 050import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET; 051import org.jdrupes.vmoperator.manager.events.ResetVm; 052import org.jdrupes.vmoperator.manager.events.VmChannel; 053import org.jdrupes.vmoperator.manager.events.VmDefChanged; 054import org.jdrupes.vmoperator.util.ExtendedObjectWrapper; 055import org.jdrupes.vmoperator.util.GsonPtr; 056import org.jgrapes.core.Channel; 057import org.jgrapes.core.Component; 058import org.jgrapes.core.annotation.Handler; 059import org.jgrapes.util.events.ConfigurationUpdate; 060 061/** 062 * Adapts Kubenetes resources for instances of the Runner 063 * application (the VMs) to changes in VM definitions (the CRs). 064 * 065 * In particular, the reconciler generates and updates: 066 * 067 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) 068 * for storage used by all VMs as a common repository for CDROM images. 069 * 070 * * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/) 071 * that defines the configuration file for the runner. 072 * 073 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) 074 * for 1 MiB of persistent storage used by the Runner (referred to as the 075 * "runnerDataPvc") 076 * 077 * * The PVCs for the VM's disks. 078 * 079 * * A [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) with the 080 * runner instance[^oldSts]. 081 * 082 * * (Optional) A load balancer 083 * [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) 084 * that allows the user to access a VM's console without knowing which 085 * node it runs on. 086 * 087 * [^oldSts]: Before version 3.4, the operator created a 088 * [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) 089 * that created the pod. 090 * 091 * The reconciler is part of the {@link Controller} component. It's 092 * configuration properties are therefore defined in 093 * ```yaml 094 * "/Manager": 095 * "/Controller": 096 * "/Reconciler": 097 * ... 098 * ``` 099 * 100 * The reconciler supports the following configuration properties: 101 * 102 * * `runnerDataPvc.storageClassName`: The storage class name 103 * to be used for the "runnerDataPvc" (the small volume used 104 * by the runner for information such as the EFI variables). By 105 * default, no `storageClassName` is generated, which causes 106 * Kubernetes to use storage from the default storage class. 107 * Define this if you want to use a specific storage class. 108 * 109 * * `cpuOvercommit`: The amount by which the current cpu count 110 * from the VM definition is divided when generating the 111 * [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) 112 * properties for the VM (defaults to 2). 113 * 114 * * `ramOvercommit`: The amount by which the current ram size 115 * from the VM definition is divided when generating the 116 * [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) 117 * properties for the VM (defaults to 1.25). 118 * 119 * * `loadBalancerService`: If defined, causes a load balancer service 120 * to be created. This property may be a boolean or 121 * YAML that defines additional labels or annotations to be merged 122 * into the service defintion. Here's an example for using 123 * [MetalLb](https://metallb.universe.tf/) as "internal load balancer": 124 * ```yaml 125 * loadBalancerService: 126 * annotations: 127 * metallb.universe.tf/loadBalancerIPs: 192.168.168.1 128 * metallb.universe.tf/ip-allocated-from-pool: single-common 129 * metallb.universe.tf/allow-shared-ip: single-common 130 * ``` 131 * This makes all VM consoles available at IP address 192.168.168.1 132 * with the port numbers from the VM definitions. 133 */ 134@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis", 135 "PMD.AvoidDuplicateLiterals" }) 136public class Reconciler extends Component { 137 138 @SuppressWarnings("PMD.SingularField") 139 private final Configuration fmConfig; 140 private final ConfigMapReconciler cmReconciler; 141 private final DisplaySecretReconciler dsReconciler; 142 private final StatefulSetReconciler stsReconciler; 143 private final PvcReconciler pvcReconciler; 144 private final PodReconciler podReconciler; 145 private final LoadBalancerReconciler lbReconciler; 146 @SuppressWarnings("PMD.UseConcurrentHashMap") 147 private final Map<String, Object> config = new HashMap<>(); 148 149 /** 150 * Instantiates a new reconciler. 151 * 152 * @param componentChannel the component channel 153 */ 154 public Reconciler(Channel componentChannel) { 155 super(componentChannel); 156 157 // Configure freemarker library 158 fmConfig = new Configuration(Configuration.VERSION_2_3_32); 159 fmConfig.setDefaultEncoding("utf-8"); 160 fmConfig.setObjectWrapper(new ExtendedObjectWrapper( 161 fmConfig.getIncompatibleImprovements())); 162 fmConfig.setTemplateExceptionHandler( 163 TemplateExceptionHandler.RETHROW_HANDLER); 164 fmConfig.setLogTemplateExceptions(false); 165 fmConfig.setClassForTemplateLoading(Reconciler.class, ""); 166 167 cmReconciler = new ConfigMapReconciler(fmConfig); 168 dsReconciler = new DisplaySecretReconciler(); 169 stsReconciler = new StatefulSetReconciler(fmConfig); 170 pvcReconciler = new PvcReconciler(fmConfig); 171 podReconciler = new PodReconciler(fmConfig); 172 lbReconciler = new LoadBalancerReconciler(fmConfig); 173 } 174 175 /** 176 * Configures the component. 177 * 178 * @param event the event 179 */ 180 @Handler 181 public void onConfigurationUpdate(ConfigurationUpdate event) { 182 event.structured(componentPath()).ifPresent(c -> { 183 config.putAll(c); 184 }); 185 } 186 187 /** 188 * Handles the change event. 189 * 190 * @param event the event 191 * @param channel the channel 192 * @throws ApiException the api exception 193 * @throws TemplateException the template exception 194 * @throws IOException Signals that an I/O exception has occurred. 195 */ 196 @Handler 197 @SuppressWarnings("PMD.ConfusingTernary") 198 public void onVmDefChanged(VmDefChanged event, VmChannel channel) 199 throws ApiException, TemplateException, IOException { 200 // We're only interested in "spec" changes. 201 if (!event.specChanged()) { 202 return; 203 } 204 205 // Ownership relationships takes care of deletions 206 var defMeta = event.vmDefinition().getMetadata(); 207 if (event.type() == K8sObserver.ResponseType.DELETED) { 208 logger.fine(() -> "VM \"" + defMeta.getName() + "\" deleted"); 209 return; 210 } 211 212 // Reconcile, use "augmented" vm definition for model 213 Map<String, Object> model 214 = prepareModel(channel.client(), patchCr(event.vmDefinition())); 215 var configMap = cmReconciler.reconcile(model, channel); 216 model.put("cm", configMap.getRaw()); 217 dsReconciler.reconcile(event, model, channel); 218 // Manage (eventual) removal of stateful set. 219 stsReconciler.reconcile(event, model, channel); 220 pvcReconciler.reconcile(event, model, channel); 221 podReconciler.reconcile(event, model, channel); 222 lbReconciler.reconcile(event, model, channel); 223 } 224 225 /** 226 * Reset the VM by incrementing the reset count and doing a 227 * partial reconcile (configmap only). 228 * 229 * @param event the event 230 * @param channel the channel 231 * @throws IOException 232 * @throws ApiException 233 * @throws TemplateException 234 */ 235 @Handler 236 public void onResetVm(ResetVm event, VmChannel channel) 237 throws ApiException, IOException, TemplateException { 238 var defRoot 239 = GsonPtr.to(channel.vmDefinition().data()).get(JsonObject.class); 240 defRoot.addProperty("resetCount", 241 defRoot.get("resetCount").getAsLong() + 1); 242 Map<String, Object> model 243 = prepareModel(channel.client(), patchCr(channel.vmDefinition())); 244 cmReconciler.reconcile(model, channel); 245 } 246 247 private DynamicKubernetesObject patchCr(K8sDynamicModel vmDef) { 248 var json = vmDef.data().deepCopy(); 249 // Adjust cdromImage path 250 adjustCdRomPaths(json); 251 252 // Adjust cloud-init data 253 adjustCloudInitData(json); 254 255 return new DynamicKubernetesObject(json); 256 } 257 258 @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") 259 private void adjustCdRomPaths(JsonObject json) { 260 var disks 261 = GsonPtr.to(json).to("spec", "vm", "disks").get(JsonArray.class); 262 for (var disk : disks) { 263 var cdrom = (JsonObject) ((JsonObject) disk).get("cdrom"); 264 if (cdrom == null) { 265 continue; 266 } 267 String image = cdrom.get("image").getAsString(); 268 if (image.isEmpty()) { 269 continue; 270 } 271 try { 272 @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") 273 var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH 274 + "/").resolve(image); 275 if ("file".equals(imageUri.getScheme())) { 276 cdrom.addProperty("image", imageUri.getPath()); 277 } else { 278 cdrom.addProperty("image", imageUri.toString()); 279 } 280 } catch (URISyntaxException e) { 281 logger.warning(() -> "Invalid CDROM image: " + image); 282 } 283 } 284 } 285 286 private void adjustCloudInitData(JsonObject json) { 287 var spec = GsonPtr.to(json).to("spec").get(JsonObject.class); 288 if (!spec.has("cloudInit")) { 289 return; 290 } 291 var metaData = GsonPtr.to(spec).to("cloudInit", "metaData"); 292 if (metaData.getAsString("instance-id").isEmpty()) { 293 metaData.set("instance-id", 294 GsonPtr.to(json).getAsString("metadata", "resourceVersion") 295 .map(s -> "v" + s).orElse("v1")); 296 } 297 if (metaData.getAsString("local-hostname").isEmpty()) { 298 metaData.set("local-hostname", 299 GsonPtr.to(json).getAsString("metadata", "name").get()); 300 } 301 } 302 303 @SuppressWarnings("PMD.CognitiveComplexity") 304 private Map<String, Object> prepareModel(K8sClient client, 305 DynamicKubernetesObject vmDef) 306 throws TemplateModelException, ApiException { 307 @SuppressWarnings("PMD.UseConcurrentHashMap") 308 Map<String, Object> model = new HashMap<>(); 309 model.put("managerVersion", 310 Optional.ofNullable(Reconciler.class.getPackage() 311 .getImplementationVersion()).orElse("(Unknown)")); 312 model.put("cr", vmDef.getRaw()); 313 model.put("constants", 314 (TemplateHashModel) new DefaultObjectWrapperBuilder( 315 Configuration.VERSION_2_3_32) 316 .build().getStaticModels() 317 .get(Constants.class.getName())); 318 model.put("reconciler", config); 319 320 // Check if we have a display secret 321 ListOptions options = new ListOptions(); 322 options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + "," 323 + "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + "," 324 + "app.kubernetes.io/instance=" + vmDef.getMetadata().getName()); 325 var dsStub = K8sV1SecretStub 326 .list(client, vmDef.getMetadata().getNamespace(), options).stream() 327 .findFirst(); 328 if (dsStub.isPresent()) { 329 dsStub.get().model().ifPresent(m -> { 330 model.put("displaySecret", m.getMetadata().getName()); 331 }); 332 } 333 334 // Methods 335 model.put("parseQuantity", new TemplateMethodModelEx() { 336 @Override 337 @SuppressWarnings("PMD.PreserveStackTrace") 338 public Object exec(@SuppressWarnings("rawtypes") List arguments) 339 throws TemplateModelException { 340 var arg = arguments.get(0); 341 if (arg instanceof Number number) { 342 return number; 343 } 344 try { 345 return Quantity.fromString(arg.toString()).getNumber(); 346 } catch (NumberFormatException e) { 347 throw new TemplateModelException("Cannot parse memory " 348 + "specified as \"" + arg + "\": " + e.getMessage()); 349 } 350 } 351 }); 352 model.put("formatMemory", new TemplateMethodModelEx() { 353 @Override 354 @SuppressWarnings("PMD.PreserveStackTrace") 355 public Object exec(@SuppressWarnings("rawtypes") List arguments) 356 throws TemplateModelException { 357 var arg = arguments.get(0); 358 if (arg instanceof SimpleNumber number) { 359 arg = number.getAsNumber(); 360 } 361 BigInteger bigInt; 362 if (arg instanceof BigInteger value) { 363 bigInt = value; 364 } else if (arg instanceof BigDecimal dec) { 365 try { 366 bigInt = dec.toBigIntegerExact(); 367 } catch (ArithmeticException e) { 368 return arg; 369 } 370 } else if (arg instanceof Integer value) { 371 bigInt = BigInteger.valueOf(value); 372 } else if (arg instanceof Long value) { 373 bigInt = BigInteger.valueOf(value); 374 } else { 375 return arg; 376 } 377 return Convertions.formatMemory(bigInt); 378 } 379 }); 380 return model; 381 } 382}