001/*
002 * VM-Operator
003 * Copyright (C) 2023 Michael N. Lipp
004 * 
005 * This program is free software: you can redistribute it and/or modify
006 * it under the terms of the GNU Affero General Public License as
007 * published by the Free Software Foundation, either version 3 of the
008 * License, or (at your option) any later version.
009 *
010 * This program is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
013 * GNU Affero General Public License for more details.
014 *
015 * You should have received a copy of the GNU Affero General Public License
016 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
017 */
018
019package org.jdrupes.vmoperator.manager;
020
021import com.google.gson.JsonArray;
022import com.google.gson.JsonObject;
023import freemarker.template.Configuration;
024import freemarker.template.DefaultObjectWrapperBuilder;
025import freemarker.template.SimpleNumber;
026import freemarker.template.TemplateException;
027import freemarker.template.TemplateExceptionHandler;
028import freemarker.template.TemplateHashModel;
029import freemarker.template.TemplateMethodModelEx;
030import freemarker.template.TemplateModelException;
031import io.kubernetes.client.custom.Quantity;
032import io.kubernetes.client.openapi.ApiException;
033import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject;
034import io.kubernetes.client.util.generic.options.ListOptions;
035import java.io.IOException;
036import java.math.BigDecimal;
037import java.math.BigInteger;
038import java.net.URI;
039import java.net.URISyntaxException;
040import java.util.HashMap;
041import java.util.List;
042import java.util.Map;
043import java.util.Optional;
044import static org.jdrupes.vmoperator.common.Constants.APP_NAME;
045import org.jdrupes.vmoperator.common.Convertions;
046import org.jdrupes.vmoperator.common.K8sClient;
047import org.jdrupes.vmoperator.common.K8sDynamicModel;
048import org.jdrupes.vmoperator.common.K8sObserver;
049import org.jdrupes.vmoperator.common.K8sV1SecretStub;
050import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET;
051import org.jdrupes.vmoperator.manager.events.ResetVm;
052import org.jdrupes.vmoperator.manager.events.VmChannel;
053import org.jdrupes.vmoperator.manager.events.VmDefChanged;
054import org.jdrupes.vmoperator.util.ExtendedObjectWrapper;
055import org.jdrupes.vmoperator.util.GsonPtr;
056import org.jgrapes.core.Channel;
057import org.jgrapes.core.Component;
058import org.jgrapes.core.annotation.Handler;
059import org.jgrapes.util.events.ConfigurationUpdate;
060
061/**
062 * Adapts Kubenetes resources for instances of the Runner 
063 * application (the VMs) to changes in VM definitions (the CRs). 
064 * 
065 * In particular, the reconciler generates and updates:  
066 * 
067 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
068 *   for storage used by all VMs as a common repository for CDROM images.
069 * 
070 * * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/)
071 *   that defines the configuration file for the runner.
072 *
073 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
074 *   for 1 MiB of persistent storage used by the Runner (referred to as the
075 *   "runnerDataPvc")
076 *   
077 * * The PVCs for the VM's disks.
078 *
079 * * A [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) with the
080 *   runner instance[^oldSts].
081 *
082 * * (Optional) A load balancer
083 *   [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/)
084 *   that allows the user to access a VM's console without knowing which
085 *   node it runs on.
086 * 
087 * [^oldSts]: Before version 3.4, the operator created a
088 *    [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
089 *    that created the pod.
090 * 
091 * The reconciler is part of the {@link Controller} component. It's 
092 * configuration properties are therefore defined in
093 * ```yaml
094 * "/Manager":
095 *   "/Controller":
096 *     "/Reconciler":
097 *       ...
098 * ```
099 *   
100 * The reconciler supports the following configuration properties:
101 * 
102 * * `runnerDataPvc.storageClassName`: The storage class name
103 *   to be used for the "runnerDataPvc" (the small volume used
104 *   by the runner for information such as the EFI variables). By
105 *   default, no `storageClassName` is generated, which causes
106 *   Kubernetes to use storage from the default storage class.
107 *   Define this if you want to use a specific storage class.
108 *
109 * * `cpuOvercommit`: The amount by which the current cpu count
110 *   from the VM definition is divided when generating the 
111 *   [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources)
112 *   properties for the VM (defaults to 2).
113 * 
114 * * `ramOvercommit`: The amount by which the current ram size
115 *   from the VM definition is divided when generating the
116 *   [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources)
117 *   properties for the VM (defaults to 1.25).
118 *   
119 * * `loadBalancerService`: If defined, causes a load balancer service 
120 *   to be created. This property may be a boolean or
121 *   YAML that defines additional labels or annotations to be merged
122 *   into the service defintion. Here's an example for using
123 *   [MetalLb](https://metallb.universe.tf/) as "internal load balancer":
124 *   ```yaml 
125 *   loadBalancerService:
126 *     annotations:
127 *       metallb.universe.tf/loadBalancerIPs: 192.168.168.1
128 *       metallb.universe.tf/ip-allocated-from-pool: single-common
129 *       metallb.universe.tf/allow-shared-ip: single-common
130 *   ```
131 *   This makes all VM consoles available at IP address 192.168.168.1
132 *   with the port numbers from the VM definitions.
133 */
134@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis",
135    "PMD.AvoidDuplicateLiterals" })
136public class Reconciler extends Component {
137
138    @SuppressWarnings("PMD.SingularField")
139    private final Configuration fmConfig;
140    private final ConfigMapReconciler cmReconciler;
141    private final DisplaySecretReconciler dsReconciler;
142    private final StatefulSetReconciler stsReconciler;
143    private final PvcReconciler pvcReconciler;
144    private final PodReconciler podReconciler;
145    private final LoadBalancerReconciler lbReconciler;
146    @SuppressWarnings("PMD.UseConcurrentHashMap")
147    private final Map<String, Object> config = new HashMap<>();
148
149    /**
150     * Instantiates a new reconciler.
151     *
152     * @param componentChannel the component channel
153     */
154    public Reconciler(Channel componentChannel) {
155        super(componentChannel);
156
157        // Configure freemarker library
158        fmConfig = new Configuration(Configuration.VERSION_2_3_32);
159        fmConfig.setDefaultEncoding("utf-8");
160        fmConfig.setObjectWrapper(new ExtendedObjectWrapper(
161            fmConfig.getIncompatibleImprovements()));
162        fmConfig.setTemplateExceptionHandler(
163            TemplateExceptionHandler.RETHROW_HANDLER);
164        fmConfig.setLogTemplateExceptions(false);
165        fmConfig.setClassForTemplateLoading(Reconciler.class, "");
166
167        cmReconciler = new ConfigMapReconciler(fmConfig);
168        dsReconciler = new DisplaySecretReconciler();
169        stsReconciler = new StatefulSetReconciler(fmConfig);
170        pvcReconciler = new PvcReconciler(fmConfig);
171        podReconciler = new PodReconciler(fmConfig);
172        lbReconciler = new LoadBalancerReconciler(fmConfig);
173    }
174
175    /**
176     * Configures the component.
177     *
178     * @param event the event
179     */
180    @Handler
181    public void onConfigurationUpdate(ConfigurationUpdate event) {
182        event.structured(componentPath()).ifPresent(c -> {
183            config.putAll(c);
184        });
185    }
186
187    /**
188     * Handles the change event.
189     *
190     * @param event the event
191     * @param channel the channel
192     * @throws ApiException the api exception
193     * @throws TemplateException the template exception
194     * @throws IOException Signals that an I/O exception has occurred.
195     */
196    @Handler
197    @SuppressWarnings("PMD.ConfusingTernary")
198    public void onVmDefChanged(VmDefChanged event, VmChannel channel)
199            throws ApiException, TemplateException, IOException {
200        // We're only interested in "spec" changes.
201        if (!event.specChanged()) {
202            return;
203        }
204
205        // Ownership relationships takes care of deletions
206        var defMeta = event.vmDefinition().getMetadata();
207        if (event.type() == K8sObserver.ResponseType.DELETED) {
208            logger.fine(() -> "VM \"" + defMeta.getName() + "\" deleted");
209            return;
210        }
211
212        // Reconcile, use "augmented" vm definition for model
213        Map<String, Object> model
214            = prepareModel(channel.client(), patchCr(event.vmDefinition()));
215        var configMap = cmReconciler.reconcile(model, channel);
216        model.put("cm", configMap.getRaw());
217        dsReconciler.reconcile(event, model, channel);
218        // Manage (eventual) removal of stateful set.
219        stsReconciler.reconcile(event, model, channel);
220        pvcReconciler.reconcile(event, model, channel);
221        podReconciler.reconcile(event, model, channel);
222        lbReconciler.reconcile(event, model, channel);
223    }
224
225    /**
226     * Reset the VM by incrementing the reset count and doing a 
227     * partial reconcile (configmap only).
228     *
229     * @param event the event
230     * @param channel the channel
231     * @throws IOException 
232     * @throws ApiException 
233     * @throws TemplateException 
234     */
235    @Handler
236    public void onResetVm(ResetVm event, VmChannel channel)
237            throws ApiException, IOException, TemplateException {
238        var defRoot
239            = GsonPtr.to(channel.vmDefinition().data()).get(JsonObject.class);
240        defRoot.addProperty("resetCount",
241            defRoot.get("resetCount").getAsLong() + 1);
242        Map<String, Object> model
243            = prepareModel(channel.client(), patchCr(channel.vmDefinition()));
244        cmReconciler.reconcile(model, channel);
245    }
246
247    private DynamicKubernetesObject patchCr(K8sDynamicModel vmDef) {
248        var json = vmDef.data().deepCopy();
249        // Adjust cdromImage path
250        adjustCdRomPaths(json);
251
252        // Adjust cloud-init data
253        adjustCloudInitData(json);
254
255        return new DynamicKubernetesObject(json);
256    }
257
258    @SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
259    private void adjustCdRomPaths(JsonObject json) {
260        var disks
261            = GsonPtr.to(json).to("spec", "vm", "disks").get(JsonArray.class);
262        for (var disk : disks) {
263            var cdrom = (JsonObject) ((JsonObject) disk).get("cdrom");
264            if (cdrom == null) {
265                continue;
266            }
267            String image = cdrom.get("image").getAsString();
268            if (image.isEmpty()) {
269                continue;
270            }
271            try {
272                @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
273                var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH
274                    + "/").resolve(image);
275                if ("file".equals(imageUri.getScheme())) {
276                    cdrom.addProperty("image", imageUri.getPath());
277                } else {
278                    cdrom.addProperty("image", imageUri.toString());
279                }
280            } catch (URISyntaxException e) {
281                logger.warning(() -> "Invalid CDROM image: " + image);
282            }
283        }
284    }
285
286    private void adjustCloudInitData(JsonObject json) {
287        var spec = GsonPtr.to(json).to("spec").get(JsonObject.class);
288        if (!spec.has("cloudInit")) {
289            return;
290        }
291        var metaData = GsonPtr.to(spec).to("cloudInit", "metaData");
292        if (metaData.getAsString("instance-id").isEmpty()) {
293            metaData.set("instance-id",
294                GsonPtr.to(json).getAsString("metadata", "resourceVersion")
295                    .map(s -> "v" + s).orElse("v1"));
296        }
297        if (metaData.getAsString("local-hostname").isEmpty()) {
298            metaData.set("local-hostname",
299                GsonPtr.to(json).getAsString("metadata", "name").get());
300        }
301    }
302
303    @SuppressWarnings("PMD.CognitiveComplexity")
304    private Map<String, Object> prepareModel(K8sClient client,
305            DynamicKubernetesObject vmDef)
306            throws TemplateModelException, ApiException {
307        @SuppressWarnings("PMD.UseConcurrentHashMap")
308        Map<String, Object> model = new HashMap<>();
309        model.put("managerVersion",
310            Optional.ofNullable(Reconciler.class.getPackage()
311                .getImplementationVersion()).orElse("(Unknown)"));
312        model.put("cr", vmDef.getRaw());
313        model.put("constants",
314            (TemplateHashModel) new DefaultObjectWrapperBuilder(
315                Configuration.VERSION_2_3_32)
316                    .build().getStaticModels()
317                    .get(Constants.class.getName()));
318        model.put("reconciler", config);
319
320        // Check if we have a display secret
321        ListOptions options = new ListOptions();
322        options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + ","
323            + "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + ","
324            + "app.kubernetes.io/instance=" + vmDef.getMetadata().getName());
325        var dsStub = K8sV1SecretStub
326            .list(client, vmDef.getMetadata().getNamespace(), options).stream()
327            .findFirst();
328        if (dsStub.isPresent()) {
329            dsStub.get().model().ifPresent(m -> {
330                model.put("displaySecret", m.getMetadata().getName());
331            });
332        }
333
334        // Methods
335        model.put("parseQuantity", new TemplateMethodModelEx() {
336            @Override
337            @SuppressWarnings("PMD.PreserveStackTrace")
338            public Object exec(@SuppressWarnings("rawtypes") List arguments)
339                    throws TemplateModelException {
340                var arg = arguments.get(0);
341                if (arg instanceof Number number) {
342                    return number;
343                }
344                try {
345                    return Quantity.fromString(arg.toString()).getNumber();
346                } catch (NumberFormatException e) {
347                    throw new TemplateModelException("Cannot parse memory "
348                        + "specified as \"" + arg + "\": " + e.getMessage());
349                }
350            }
351        });
352        model.put("formatMemory", new TemplateMethodModelEx() {
353            @Override
354            @SuppressWarnings("PMD.PreserveStackTrace")
355            public Object exec(@SuppressWarnings("rawtypes") List arguments)
356                    throws TemplateModelException {
357                var arg = arguments.get(0);
358                if (arg instanceof SimpleNumber number) {
359                    arg = number.getAsNumber();
360                }
361                BigInteger bigInt;
362                if (arg instanceof BigInteger value) {
363                    bigInt = value;
364                } else if (arg instanceof BigDecimal dec) {
365                    try {
366                        bigInt = dec.toBigIntegerExact();
367                    } catch (ArithmeticException e) {
368                        return arg;
369                    }
370                } else if (arg instanceof Integer value) {
371                    bigInt = BigInteger.valueOf(value);
372                } else if (arg instanceof Long value) {
373                    bigInt = BigInteger.valueOf(value);
374                } else {
375                    return arg;
376                }
377                return Convertions.formatMemory(bigInt);
378            }
379        });
380        return model;
381    }
382}