001/*
002 * VM-Operator
003 * Copyright (C) 2023 Michael N. Lipp
004 * 
005 * This program is free software: you can redistribute it and/or modify
006 * it under the terms of the GNU Affero General Public License as
007 * published by the Free Software Foundation, either version 3 of the
008 * License, or (at your option) any later version.
009 *
010 * This program is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
013 * GNU Affero General Public License for more details.
014 *
015 * You should have received a copy of the GNU Affero General Public License
016 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
017 */
018
019package org.jdrupes.vmoperator.manager;
020
021import com.google.gson.JsonArray;
022import com.google.gson.JsonObject;
023import freemarker.core.ParseException;
024import freemarker.template.Configuration;
025import freemarker.template.DefaultObjectWrapperBuilder;
026import freemarker.template.MalformedTemplateNameException;
027import freemarker.template.SimpleNumber;
028import freemarker.template.TemplateException;
029import freemarker.template.TemplateExceptionHandler;
030import freemarker.template.TemplateHashModel;
031import freemarker.template.TemplateMethodModelEx;
032import freemarker.template.TemplateModelException;
033import freemarker.template.TemplateNotFoundException;
034import io.kubernetes.client.custom.Quantity;
035import io.kubernetes.client.openapi.ApiException;
036import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject;
037import io.kubernetes.client.util.generic.options.ListOptions;
038import java.io.IOException;
039import java.math.BigDecimal;
040import java.math.BigInteger;
041import java.net.URI;
042import java.net.URISyntaxException;
043import java.util.HashMap;
044import java.util.List;
045import java.util.Map;
046import java.util.Optional;
047import static org.jdrupes.vmoperator.common.Constants.APP_NAME;
048import org.jdrupes.vmoperator.common.Convertions;
049import org.jdrupes.vmoperator.common.K8sClient;
050import org.jdrupes.vmoperator.common.K8sDynamicModel;
051import org.jdrupes.vmoperator.common.K8sObserver;
052import org.jdrupes.vmoperator.common.K8sV1SecretStub;
053import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET;
054import org.jdrupes.vmoperator.manager.events.ResetVm;
055import org.jdrupes.vmoperator.manager.events.VmChannel;
056import org.jdrupes.vmoperator.manager.events.VmDefChanged;
057import org.jdrupes.vmoperator.util.ExtendedObjectWrapper;
058import org.jdrupes.vmoperator.util.GsonPtr;
059import org.jgrapes.core.Channel;
060import org.jgrapes.core.Component;
061import org.jgrapes.core.annotation.Handler;
062import org.jgrapes.util.events.ConfigurationUpdate;
063
064/**
065 * Adapts Kubenetes resources for instances of the Runner 
066 * application (the VMs) to changes in VM definitions (the CRs). 
067 * 
068 * In particular, the reconciler generates and updates:  
069 * 
070 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
071 *   for storage used by all VMs as a common repository for CDROM images.
072 * 
073 * * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/)
074 *   that defines the configuration file for the runner.
075 *    
076 * * A [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
077 *   that creates 
078 *   * the [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) 
079 *     with the Runner instance, 
080 *   * a PVC for 1 MiB of persistent storage used by the Runner 
081 *     (referred to as the "runnerDataPvc") and
082 *   * the PVCs for the VM's disks.
083 *    
084 * * (Optional) A load balancer
085 *   [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/)
086 *   that allows the user to access a VM's console without knowing which
087 *   node it runs on.
088 * 
089 * The reconciler is part of the {@link Controller} component. It's 
090 * configuration properties are therefore defined in
091 * ```yaml
092 * "/Manager":
093 *   "/Controller":
094 *     "/Reconciler":
095 *       ...
096 * ```
097 *   
098 * The reconciler supports the following configuration properties:
099 * 
100 * * `runnerDataPvc.storageClassName`: The storage class name
101 *   to be used for the "runnerDataPvc" (the small volume used
102 *   by the runner for information such as the EFI variables). By
103 *   default, no `storageClassName` is generated, which causes
104 *   Kubernetes to use storage from the default storage class.
105 *   Define this if you want to use a specific storage class.
106 *
107 * * `cpuOvercommit`: The amount by which the current cpu count
108 *   from the VM definition is divided when generating the 
109 *   [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources)
110 *   properties for the VM (defaults to 2).
111 * 
112 * * `ramOvercommit`: The amount by which the current ram size
113 *   from the VM definition is divided when generating the
114 *   [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources)
115 *   properties for the VM (defaults to 1.25).
116 *   
117 * * `loadBalancerService`: If defined, causes a load balancer service 
118 *   to be created. This property may be a boolean or
119 *   YAML that defines additional labels or annotations to be merged
120 *   into the service defintion. Here's an example for using
121 *   [MetalLb](https://metallb.universe.tf/) as "internal load balancer":
122 *   ```yaml 
123 *   loadBalancerService:
124 *     annotations:
125 *       metallb.universe.tf/loadBalancerIPs: 192.168.168.1
126 *       metallb.universe.tf/ip-allocated-from-pool: single-common
127 *       metallb.universe.tf/allow-shared-ip: single-common
128 *   ```
129 *   This makes all VM consoles available at IP address 192.168.168.1
130 *   with the port numbers from the VM definitions.
131 */
132@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis",
133    "PMD.AvoidDuplicateLiterals" })
134public class Reconciler extends Component {
135
136    @SuppressWarnings("PMD.SingularField")
137    private final Configuration fmConfig;
138    private final ConfigMapReconciler cmReconciler;
139    private final DisplaySecretReconciler dsReconciler;
140    private final StatefulSetReconciler stsReconciler;
141    private final LoadBalancerReconciler lbReconciler;
142    @SuppressWarnings("PMD.UseConcurrentHashMap")
143    private final Map<String, Object> config = new HashMap<>();
144
145    /**
146     * Instantiates a new reconciler.
147     *
148     * @param componentChannel the component channel
149     */
150    public Reconciler(Channel componentChannel) {
151        super(componentChannel);
152
153        // Configure freemarker library
154        fmConfig = new Configuration(Configuration.VERSION_2_3_32);
155        fmConfig.setDefaultEncoding("utf-8");
156        fmConfig.setObjectWrapper(new ExtendedObjectWrapper(
157            fmConfig.getIncompatibleImprovements()));
158        fmConfig.setTemplateExceptionHandler(
159            TemplateExceptionHandler.RETHROW_HANDLER);
160        fmConfig.setLogTemplateExceptions(false);
161        fmConfig.setClassForTemplateLoading(Reconciler.class, "");
162
163        cmReconciler = new ConfigMapReconciler(fmConfig);
164        dsReconciler = new DisplaySecretReconciler();
165        stsReconciler = new StatefulSetReconciler(fmConfig);
166        lbReconciler = new LoadBalancerReconciler(fmConfig);
167    }
168
169    /**
170     * Configures the component.
171     *
172     * @param event the event
173     */
174    @Handler
175    public void onConfigurationUpdate(ConfigurationUpdate event) {
176        event.structured(componentPath()).ifPresent(c -> {
177            config.putAll(c);
178        });
179    }
180
181    /**
182     * Handles the change event.
183     *
184     * @param event the event
185     * @param channel the channel
186     * @throws ApiException the api exception
187     * @throws TemplateException the template exception
188     * @throws IOException Signals that an I/O exception has occurred.
189     */
190    @Handler
191    @SuppressWarnings("PMD.ConfusingTernary")
192    public void onVmDefChanged(VmDefChanged event, VmChannel channel)
193            throws ApiException, TemplateException, IOException {
194        // We're only interested in "spec" changes.
195        if (!event.specChanged()) {
196            return;
197        }
198
199        // Ownership relationships takes care of deletions
200        var defMeta = event.vmDefinition().getMetadata();
201        if (event.type() == K8sObserver.ResponseType.DELETED) {
202            logger.fine(() -> "VM \"" + defMeta.getName() + "\" deleted");
203            return;
204        }
205
206        // Reconcile, use "augmented" vm definition for model
207        Map<String, Object> model
208            = prepareModel(channel.client(), patchCr(event.vmDefinition()));
209        var configMap = cmReconciler.reconcile(model, channel);
210        model.put("cm", configMap.getRaw());
211        dsReconciler.reconcile(event, model, channel);
212        stsReconciler.reconcile(event, model, channel);
213        lbReconciler.reconcile(event, model, channel);
214    }
215
216    /**
217     * Reset the VM by incrementing the reset count and doing a 
218     * partial reconcile (configmap only).
219     *
220     * @param event the event
221     * @param channel the channel
222     * @throws IOException 
223     * @throws ApiException 
224     * @throws TemplateException 
225     */
226    @Handler
227    public void onResetVm(ResetVm event, VmChannel channel)
228            throws ApiException, IOException, TemplateException {
229        var defRoot
230            = GsonPtr.to(channel.vmDefinition().data()).get(JsonObject.class);
231        defRoot.addProperty("resetCount",
232            defRoot.get("resetCount").getAsLong() + 1);
233        Map<String, Object> model
234            = prepareModel(channel.client(), patchCr(channel.vmDefinition()));
235        cmReconciler.reconcile(model, channel);
236    }
237
238    private DynamicKubernetesObject patchCr(K8sDynamicModel vmDef) {
239        var json = vmDef.data().deepCopy();
240        // Adjust cdromImage path
241        adjustCdRomPaths(json);
242
243        // Adjust cloud-init data
244        adjustCloudInitData(json);
245
246        return new DynamicKubernetesObject(json);
247    }
248
249    @SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
250    private void adjustCdRomPaths(JsonObject json) {
251        var disks
252            = GsonPtr.to(json).to("spec", "vm", "disks").get(JsonArray.class);
253        for (var disk : disks) {
254            var cdrom = (JsonObject) ((JsonObject) disk).get("cdrom");
255            if (cdrom == null) {
256                continue;
257            }
258            String image = cdrom.get("image").getAsString();
259            if (image.isEmpty()) {
260                continue;
261            }
262            try {
263                @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
264                var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH
265                    + "/").resolve(image);
266                if ("file".equals(imageUri.getScheme())) {
267                    cdrom.addProperty("image", imageUri.getPath());
268                } else {
269                    cdrom.addProperty("image", imageUri.toString());
270                }
271            } catch (URISyntaxException e) {
272                logger.warning(() -> "Invalid CDROM image: " + image);
273            }
274        }
275    }
276
277    private void adjustCloudInitData(JsonObject json) {
278        var spec = GsonPtr.to(json).to("spec").get(JsonObject.class);
279        if (!spec.has("cloudInit")) {
280            return;
281        }
282        var metaData = GsonPtr.to(spec).to("cloudInit", "metaData");
283        if (metaData.getAsString("instance-id").isEmpty()) {
284            metaData.set("instance-id",
285                GsonPtr.to(json).getAsString("metadata", "resourceVersion")
286                    .map(s -> "v" + s).orElse("v1"));
287        }
288        if (metaData.getAsString("local-hostname").isEmpty()) {
289            metaData.set("local-hostname",
290                GsonPtr.to(json).getAsString("metadata", "name").get());
291        }
292    }
293
294    @SuppressWarnings("PMD.CognitiveComplexity")
295    private Map<String, Object> prepareModel(K8sClient client,
296            DynamicKubernetesObject vmDef)
297            throws TemplateModelException, ApiException {
298        @SuppressWarnings("PMD.UseConcurrentHashMap")
299        Map<String, Object> model = new HashMap<>();
300        model.put("managerVersion",
301            Optional.ofNullable(Reconciler.class.getPackage()
302                .getImplementationVersion()).orElse("(Unknown)"));
303        model.put("cr", vmDef.getRaw());
304        model.put("constants",
305            (TemplateHashModel) new DefaultObjectWrapperBuilder(
306                Configuration.VERSION_2_3_32)
307                    .build().getStaticModels()
308                    .get(Constants.class.getName()));
309        model.put("reconciler", config);
310
311        // Check if we have a display secret
312        ListOptions options = new ListOptions();
313        options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + ","
314            + "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + ","
315            + "app.kubernetes.io/instance=" + vmDef.getMetadata().getName());
316        var dsStub = K8sV1SecretStub
317            .list(client, vmDef.getMetadata().getNamespace(), options).stream()
318            .findFirst();
319        if (dsStub.isPresent()) {
320            dsStub.get().model().ifPresent(m -> {
321                model.put("displaySecret", m.getMetadata().getName());
322            });
323        }
324
325        // Methods
326        model.put("parseQuantity", new TemplateMethodModelEx() {
327            @Override
328            @SuppressWarnings("PMD.PreserveStackTrace")
329            public Object exec(@SuppressWarnings("rawtypes") List arguments)
330                    throws TemplateModelException {
331                var arg = arguments.get(0);
332                if (arg instanceof Number number) {
333                    return number;
334                }
335                try {
336                    return Quantity.fromString(arg.toString()).getNumber();
337                } catch (NumberFormatException e) {
338                    throw new TemplateModelException("Cannot parse memory "
339                        + "specified as \"" + arg + "\": " + e.getMessage());
340                }
341            }
342        });
343        model.put("formatMemory", new TemplateMethodModelEx() {
344            @Override
345            @SuppressWarnings("PMD.PreserveStackTrace")
346            public Object exec(@SuppressWarnings("rawtypes") List arguments)
347                    throws TemplateModelException {
348                var arg = arguments.get(0);
349                if (arg instanceof SimpleNumber number) {
350                    arg = number.getAsNumber();
351                }
352                BigInteger bigInt;
353                if (arg instanceof BigInteger value) {
354                    bigInt = value;
355                } else if (arg instanceof BigDecimal dec) {
356                    try {
357                        bigInt = dec.toBigIntegerExact();
358                    } catch (ArithmeticException e) {
359                        return arg;
360                    }
361                } else if (arg instanceof Integer value) {
362                    bigInt = BigInteger.valueOf(value);
363                } else if (arg instanceof Long value) {
364                    bigInt = BigInteger.valueOf(value);
365                } else {
366                    return arg;
367                }
368                return Convertions.formatMemory(bigInt);
369            }
370        });
371        return model;
372    }
373}