001/* 002 * VM-Operator 003 * Copyright (C) 2023 Michael N. Lipp 004 * 005 * This program is free software: you can redistribute it and/or modify 006 * it under the terms of the GNU Affero General Public License as 007 * published by the Free Software Foundation, either version 3 of the 008 * License, or (at your option) any later version. 009 * 010 * This program is distributed in the hope that it will be useful, 011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 013 * GNU Affero General Public License for more details. 014 * 015 * You should have received a copy of the GNU Affero General Public License 016 * along with this program. If not, see <https://www.gnu.org/licenses/>. 017 */ 018 019package org.jdrupes.vmoperator.manager; 020 021import com.google.gson.JsonArray; 022import com.google.gson.JsonObject; 023import freemarker.template.Configuration; 024import freemarker.template.DefaultObjectWrapperBuilder; 025import freemarker.template.SimpleNumber; 026import freemarker.template.TemplateException; 027import freemarker.template.TemplateExceptionHandler; 028import freemarker.template.TemplateHashModel; 029import freemarker.template.TemplateMethodModelEx; 030import freemarker.template.TemplateModelException; 031import io.kubernetes.client.custom.Quantity; 032import io.kubernetes.client.openapi.ApiException; 033import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject; 034import io.kubernetes.client.util.generic.options.ListOptions; 035import java.io.IOException; 036import java.math.BigDecimal; 037import java.math.BigInteger; 038import java.net.URI; 039import java.net.URISyntaxException; 040import java.util.HashMap; 041import java.util.List; 042import java.util.Map; 043import java.util.Optional; 044import static org.jdrupes.vmoperator.common.Constants.APP_NAME; 045import org.jdrupes.vmoperator.common.Convertions; 046import org.jdrupes.vmoperator.common.K8sClient; 047import org.jdrupes.vmoperator.common.K8sDynamicModel; 048import org.jdrupes.vmoperator.common.K8sObserver; 049import org.jdrupes.vmoperator.common.K8sV1SecretStub; 050import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET; 051import org.jdrupes.vmoperator.manager.events.ResetVm; 052import org.jdrupes.vmoperator.manager.events.VmChannel; 053import org.jdrupes.vmoperator.manager.events.VmDefChanged; 054import org.jdrupes.vmoperator.util.ExtendedObjectWrapper; 055import org.jdrupes.vmoperator.util.GsonPtr; 056import org.jgrapes.core.Channel; 057import org.jgrapes.core.Component; 058import org.jgrapes.core.annotation.Handler; 059import org.jgrapes.util.events.ConfigurationUpdate; 060 061/** 062 * Adapts Kubenetes resources for instances of the Runner 063 * application (the VMs) to changes in VM definitions (the CRs). 064 * 065 * In particular, the reconciler generates and updates: 066 * 067 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) 068 * for storage used by all VMs as a common repository for CDROM images. 069 * 070 * * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/) 071 * that defines the configuration file for the runner. 072 * 073 * * A [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) 074 * that creates 075 * * the [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) 076 * with the Runner instance, 077 * * a PVC for 1 MiB of persistent storage used by the Runner 078 * (referred to as the "runnerDataPvc") and 079 * * the PVCs for the VM's disks. 080 * 081 * * (Optional) A load balancer 082 * [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) 083 * that allows the user to access a VM's console without knowing which 084 * node it runs on. 085 * 086 * The reconciler is part of the {@link Controller} component. It's 087 * configuration properties are therefore defined in 088 * ```yaml 089 * "/Manager": 090 * "/Controller": 091 * "/Reconciler": 092 * ... 093 * ``` 094 * 095 * The reconciler supports the following configuration properties: 096 * 097 * * `runnerDataPvc.storageClassName`: The storage class name 098 * to be used for the "runnerDataPvc" (the small volume used 099 * by the runner for information such as the EFI variables). By 100 * default, no `storageClassName` is generated, which causes 101 * Kubernetes to use storage from the default storage class. 102 * Define this if you want to use a specific storage class. 103 * 104 * * `cpuOvercommit`: The amount by which the current cpu count 105 * from the VM definition is divided when generating the 106 * [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) 107 * properties for the VM (defaults to 2). 108 * 109 * * `ramOvercommit`: The amount by which the current ram size 110 * from the VM definition is divided when generating the 111 * [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) 112 * properties for the VM (defaults to 1.25). 113 * 114 * * `loadBalancerService`: If defined, causes a load balancer service 115 * to be created. This property may be a boolean or 116 * YAML that defines additional labels or annotations to be merged 117 * into the service defintion. Here's an example for using 118 * [MetalLb](https://metallb.universe.tf/) as "internal load balancer": 119 * ```yaml 120 * loadBalancerService: 121 * annotations: 122 * metallb.universe.tf/loadBalancerIPs: 192.168.168.1 123 * metallb.universe.tf/ip-allocated-from-pool: single-common 124 * metallb.universe.tf/allow-shared-ip: single-common 125 * ``` 126 * This makes all VM consoles available at IP address 192.168.168.1 127 * with the port numbers from the VM definitions. 128 */ 129@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis", 130 "PMD.AvoidDuplicateLiterals" }) 131public class Reconciler extends Component { 132 133 @SuppressWarnings("PMD.SingularField") 134 private final Configuration fmConfig; 135 private final ConfigMapReconciler cmReconciler; 136 private final DisplaySecretReconciler dsReconciler; 137 private final StatefulSetReconciler stsReconciler; 138 private final LoadBalancerReconciler lbReconciler; 139 @SuppressWarnings("PMD.UseConcurrentHashMap") 140 private final Map<String, Object> config = new HashMap<>(); 141 142 /** 143 * Instantiates a new reconciler. 144 * 145 * @param componentChannel the component channel 146 */ 147 public Reconciler(Channel componentChannel) { 148 super(componentChannel); 149 150 // Configure freemarker library 151 fmConfig = new Configuration(Configuration.VERSION_2_3_32); 152 fmConfig.setDefaultEncoding("utf-8"); 153 fmConfig.setObjectWrapper(new ExtendedObjectWrapper( 154 fmConfig.getIncompatibleImprovements())); 155 fmConfig.setTemplateExceptionHandler( 156 TemplateExceptionHandler.RETHROW_HANDLER); 157 fmConfig.setLogTemplateExceptions(false); 158 fmConfig.setClassForTemplateLoading(Reconciler.class, ""); 159 160 cmReconciler = new ConfigMapReconciler(fmConfig); 161 dsReconciler = new DisplaySecretReconciler(); 162 stsReconciler = new StatefulSetReconciler(fmConfig); 163 lbReconciler = new LoadBalancerReconciler(fmConfig); 164 } 165 166 /** 167 * Configures the component. 168 * 169 * @param event the event 170 */ 171 @Handler 172 public void onConfigurationUpdate(ConfigurationUpdate event) { 173 event.structured(componentPath()).ifPresent(c -> { 174 config.putAll(c); 175 }); 176 } 177 178 /** 179 * Handles the change event. 180 * 181 * @param event the event 182 * @param channel the channel 183 * @throws ApiException the api exception 184 * @throws TemplateException the template exception 185 * @throws IOException Signals that an I/O exception has occurred. 186 */ 187 @Handler 188 @SuppressWarnings("PMD.ConfusingTernary") 189 public void onVmDefChanged(VmDefChanged event, VmChannel channel) 190 throws ApiException, TemplateException, IOException { 191 // We're only interested in "spec" changes. 192 if (!event.specChanged()) { 193 return; 194 } 195 196 // Ownership relationships takes care of deletions 197 var defMeta = event.vmDefinition().getMetadata(); 198 if (event.type() == K8sObserver.ResponseType.DELETED) { 199 logger.fine(() -> "VM \"" + defMeta.getName() + "\" deleted"); 200 return; 201 } 202 203 // Reconcile, use "augmented" vm definition for model 204 Map<String, Object> model 205 = prepareModel(channel.client(), patchCr(event.vmDefinition())); 206 var configMap = cmReconciler.reconcile(model, channel); 207 model.put("cm", configMap.getRaw()); 208 dsReconciler.reconcile(event, model, channel); 209 stsReconciler.reconcile(event, model, channel); 210 lbReconciler.reconcile(event, model, channel); 211 } 212 213 /** 214 * Reset the VM by incrementing the reset count and doing a 215 * partial reconcile (configmap only). 216 * 217 * @param event the event 218 * @param channel the channel 219 * @throws IOException 220 * @throws ApiException 221 * @throws TemplateException 222 */ 223 @Handler 224 public void onResetVm(ResetVm event, VmChannel channel) 225 throws ApiException, IOException, TemplateException { 226 var defRoot 227 = GsonPtr.to(channel.vmDefinition().data()).get(JsonObject.class); 228 defRoot.addProperty("resetCount", 229 defRoot.get("resetCount").getAsLong() + 1); 230 Map<String, Object> model 231 = prepareModel(channel.client(), patchCr(channel.vmDefinition())); 232 cmReconciler.reconcile(model, channel); 233 } 234 235 private DynamicKubernetesObject patchCr(K8sDynamicModel vmDef) { 236 var json = vmDef.data().deepCopy(); 237 // Adjust cdromImage path 238 adjustCdRomPaths(json); 239 240 // Adjust cloud-init data 241 adjustCloudInitData(json); 242 243 return new DynamicKubernetesObject(json); 244 } 245 246 @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") 247 private void adjustCdRomPaths(JsonObject json) { 248 var disks 249 = GsonPtr.to(json).to("spec", "vm", "disks").get(JsonArray.class); 250 for (var disk : disks) { 251 var cdrom = (JsonObject) ((JsonObject) disk).get("cdrom"); 252 if (cdrom == null) { 253 continue; 254 } 255 String image = cdrom.get("image").getAsString(); 256 if (image.isEmpty()) { 257 continue; 258 } 259 try { 260 @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") 261 var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH 262 + "/").resolve(image); 263 if ("file".equals(imageUri.getScheme())) { 264 cdrom.addProperty("image", imageUri.getPath()); 265 } else { 266 cdrom.addProperty("image", imageUri.toString()); 267 } 268 } catch (URISyntaxException e) { 269 logger.warning(() -> "Invalid CDROM image: " + image); 270 } 271 } 272 } 273 274 private void adjustCloudInitData(JsonObject json) { 275 var spec = GsonPtr.to(json).to("spec").get(JsonObject.class); 276 if (!spec.has("cloudInit")) { 277 return; 278 } 279 var metaData = GsonPtr.to(spec).to("cloudInit", "metaData"); 280 if (metaData.getAsString("instance-id").isEmpty()) { 281 metaData.set("instance-id", 282 GsonPtr.to(json).getAsString("metadata", "resourceVersion") 283 .map(s -> "v" + s).orElse("v1")); 284 } 285 if (metaData.getAsString("local-hostname").isEmpty()) { 286 metaData.set("local-hostname", 287 GsonPtr.to(json).getAsString("metadata", "name").get()); 288 } 289 } 290 291 @SuppressWarnings("PMD.CognitiveComplexity") 292 private Map<String, Object> prepareModel(K8sClient client, 293 DynamicKubernetesObject vmDef) 294 throws TemplateModelException, ApiException { 295 @SuppressWarnings("PMD.UseConcurrentHashMap") 296 Map<String, Object> model = new HashMap<>(); 297 model.put("managerVersion", 298 Optional.ofNullable(Reconciler.class.getPackage() 299 .getImplementationVersion()).orElse("(Unknown)")); 300 model.put("cr", vmDef.getRaw()); 301 model.put("constants", 302 (TemplateHashModel) new DefaultObjectWrapperBuilder( 303 Configuration.VERSION_2_3_32) 304 .build().getStaticModels() 305 .get(Constants.class.getName())); 306 model.put("reconciler", config); 307 308 // Check if we have a display secret 309 ListOptions options = new ListOptions(); 310 options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + "," 311 + "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + "," 312 + "app.kubernetes.io/instance=" + vmDef.getMetadata().getName()); 313 var dsStub = K8sV1SecretStub 314 .list(client, vmDef.getMetadata().getNamespace(), options).stream() 315 .findFirst(); 316 if (dsStub.isPresent()) { 317 dsStub.get().model().ifPresent(m -> { 318 model.put("displaySecret", m.getMetadata().getName()); 319 }); 320 } 321 322 // Methods 323 model.put("parseQuantity", new TemplateMethodModelEx() { 324 @Override 325 @SuppressWarnings("PMD.PreserveStackTrace") 326 public Object exec(@SuppressWarnings("rawtypes") List arguments) 327 throws TemplateModelException { 328 var arg = arguments.get(0); 329 if (arg instanceof Number number) { 330 return number; 331 } 332 try { 333 return Quantity.fromString(arg.toString()).getNumber(); 334 } catch (NumberFormatException e) { 335 throw new TemplateModelException("Cannot parse memory " 336 + "specified as \"" + arg + "\": " + e.getMessage()); 337 } 338 } 339 }); 340 model.put("formatMemory", new TemplateMethodModelEx() { 341 @Override 342 @SuppressWarnings("PMD.PreserveStackTrace") 343 public Object exec(@SuppressWarnings("rawtypes") List arguments) 344 throws TemplateModelException { 345 var arg = arguments.get(0); 346 if (arg instanceof SimpleNumber number) { 347 arg = number.getAsNumber(); 348 } 349 BigInteger bigInt; 350 if (arg instanceof BigInteger value) { 351 bigInt = value; 352 } else if (arg instanceof BigDecimal dec) { 353 try { 354 bigInt = dec.toBigIntegerExact(); 355 } catch (ArithmeticException e) { 356 return arg; 357 } 358 } else if (arg instanceof Integer value) { 359 bigInt = BigInteger.valueOf(value); 360 } else if (arg instanceof Long value) { 361 bigInt = BigInteger.valueOf(value); 362 } else { 363 return arg; 364 } 365 return Convertions.formatMemory(bigInt); 366 } 367 }); 368 return model; 369 } 370}