Source code

001/*
002 * VM-Operator
003 * Copyright (C) 2023 Michael N. Lipp
004 * 
005 * This program is free software: you can redistribute it and/or modify
006 * it under the terms of the GNU Affero General Public License as
007 * published by the Free Software Foundation, either version 3 of the
008 * License, or (at your option) any later version.
009 *
010 * This program is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
013 * GNU Affero General Public License for more details.
014 *
015 * You should have received a copy of the GNU Affero General Public License
016 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
017 */
018
019package org.jdrupes.vmoperator.manager;
020
021import com.fasterxml.jackson.core.JsonProcessingException;
022import com.fasterxml.jackson.databind.ObjectMapper;
023import freemarker.template.AdapterTemplateModel;
024import freemarker.template.Configuration;
025import freemarker.template.DefaultObjectWrapperBuilder;
026import freemarker.template.SimpleNumber;
027import freemarker.template.SimpleScalar;
028import freemarker.template.TemplateException;
029import freemarker.template.TemplateExceptionHandler;
030import freemarker.template.TemplateHashModel;
031import freemarker.template.TemplateMethodModelEx;
032import freemarker.template.TemplateModelException;
033import io.kubernetes.client.custom.Quantity;
034import io.kubernetes.client.openapi.ApiException;
035import io.kubernetes.client.openapi.models.V1ObjectMeta;
036import io.kubernetes.client.util.generic.options.ListOptions;
037import java.io.IOException;
038import java.math.BigDecimal;
039import java.math.BigInteger;
040import java.net.URI;
041import java.net.URISyntaxException;
042import java.util.HashMap;
043import java.util.List;
044import java.util.Map;
045import java.util.Optional;
046import static org.jdrupes.vmoperator.common.Constants.APP_NAME;
047import org.jdrupes.vmoperator.common.Convertions;
048import org.jdrupes.vmoperator.common.K8sClient;
049import org.jdrupes.vmoperator.common.K8sObserver;
050import org.jdrupes.vmoperator.common.K8sV1SecretStub;
051import org.jdrupes.vmoperator.common.VmDefinition;
052import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET;
053import org.jdrupes.vmoperator.manager.events.ResetVm;
054import org.jdrupes.vmoperator.manager.events.VmChannel;
055import org.jdrupes.vmoperator.manager.events.VmDefChanged;
056import org.jdrupes.vmoperator.util.DataPath;
057import org.jdrupes.vmoperator.util.ExtendedObjectWrapper;
058import org.jgrapes.core.Channel;
059import org.jgrapes.core.Component;
060import org.jgrapes.core.annotation.Handler;
061import org.jgrapes.util.events.ConfigurationUpdate;
062
063/**
064 * Adapts Kubenetes resources for instances of the Runner 
065 * application (the VMs) to changes in VM definitions (the CRs). 
066 * 
067 * In particular, the reconciler generates and updates:  
068 * 
069 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
070 *   for storage used by all VMs as a common repository for CDROM images.
071 * 
072 * * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/)
073 *   that defines the configuration file for the runner.
074 *
075 * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
076 *   for 1 MiB of persistent storage used by the Runner (referred to as the
077 *   "runnerDataPvc")
078 *   
079 * * The PVCs for the VM's disks.
080 *
081 * * A [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) with the
082 *   runner instance[^oldSts].
083 *
084 * * (Optional) A load balancer
085 *   [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/)
086 *   that allows the user to access a VM's console without knowing which
087 *   node it runs on.
088 * 
089 * [^oldSts]: Before version 3.4, the operator created a
090 *    [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
091 *    that created the pod.
092 * 
093 * The reconciler is part of the {@link Controller} component. It's 
094 * configuration properties are therefore defined in
095 * ```yaml
096 * "/Manager":
097 *   "/Controller":
098 *     "/Reconciler":
099 *       ...
100 * ```
101 *   
102 * The reconciler supports the following configuration properties:
103 * 
104 * * `runnerDataPvc.storageClassName`: The storage class name
105 *   to be used for the "runnerDataPvc" (the small volume used
106 *   by the runner for information such as the EFI variables). By
107 *   default, no `storageClassName` is generated, which causes
108 *   Kubernetes to use storage from the default storage class.
109 *   Define this if you want to use a specific storage class.
110 *
111 * * `cpuOvercommit`: The amount by which the current cpu count
112 *   from the VM definition is divided when generating the 
113 *   [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources)
114 *   properties for the VM (defaults to 2).
115 * 
116 * * `ramOvercommit`: The amount by which the current ram size
117 *   from the VM definition is divided when generating the
118 *   [`resources`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources)
119 *   properties for the VM (defaults to 1.25).
120 *   
121 * * `loadBalancerService`: If defined, causes a load balancer service 
122 *   to be created. This property may be a boolean or
123 *   YAML that defines additional labels or annotations to be merged
124 *   into the service defintion. Here's an example for using
125 *   [MetalLb](https://metallb.universe.tf/) as "internal load balancer":
126 *   ```yaml 
127 *   loadBalancerService:
128 *     annotations:
129 *       metallb.universe.tf/loadBalancerIPs: 192.168.168.1
130 *       metallb.universe.tf/ip-allocated-from-pool: single-common
131 *       metallb.universe.tf/allow-shared-ip: single-common
132 *   ```
133 *   This makes all VM consoles available at IP address 192.168.168.1
134 *   with the port numbers from the VM definitions.
135 *   
136 * * `loggingProperties`: If defined, specifies the default logging
137 *   properties to be used by the runners managed by the controller.
138 *   This property is a string that holds the content of
139 *   a logging.properties file.
140 */
141@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis",
142    "PMD.AvoidDuplicateLiterals" })
143public class Reconciler extends Component {
144
145    /** The Constant mapper. */
146    @SuppressWarnings("PMD.FieldNamingConventions")
147    protected static final ObjectMapper mapper = new ObjectMapper();
148
149    @SuppressWarnings("PMD.SingularField")
150    private final Configuration fmConfig;
151    private final ConfigMapReconciler cmReconciler;
152    private final DisplaySecretReconciler dsReconciler;
153    private final StatefulSetReconciler stsReconciler;
154    private final PvcReconciler pvcReconciler;
155    private final PodReconciler podReconciler;
156    private final LoadBalancerReconciler lbReconciler;
157    @SuppressWarnings("PMD.UseConcurrentHashMap")
158    private final Map<String, Object> config = new HashMap<>();
159
160    /**
161     * Instantiates a new reconciler.
162     *
163     * @param componentChannel the component channel
164     */
165    public Reconciler(Channel componentChannel) {
166        super(componentChannel);
167
168        // Configure freemarker library
169        fmConfig = new Configuration(Configuration.VERSION_2_3_32);
170        fmConfig.setDefaultEncoding("utf-8");
171        fmConfig.setObjectWrapper(new ExtendedObjectWrapper(
172            fmConfig.getIncompatibleImprovements()));
173        fmConfig.setTemplateExceptionHandler(
174            TemplateExceptionHandler.RETHROW_HANDLER);
175        fmConfig.setLogTemplateExceptions(false);
176        fmConfig.setClassForTemplateLoading(Reconciler.class, "");
177
178        cmReconciler = new ConfigMapReconciler(fmConfig);
179        dsReconciler = new DisplaySecretReconciler();
180        stsReconciler = new StatefulSetReconciler(fmConfig);
181        pvcReconciler = new PvcReconciler(fmConfig);
182        podReconciler = new PodReconciler(fmConfig);
183        lbReconciler = new LoadBalancerReconciler(fmConfig);
184    }
185
186    /**
187     * Configures the component.
188     *
189     * @param event the event
190     */
191    @Handler
192    public void onConfigurationUpdate(ConfigurationUpdate event) {
193        event.structured(componentPath()).ifPresent(c -> {
194            config.putAll(c);
195        });
196    }
197
198    /**
199     * Handles the change event.
200     *
201     * @param event the event
202     * @param channel the channel
203     * @throws ApiException the api exception
204     * @throws TemplateException the template exception
205     * @throws IOException Signals that an I/O exception has occurred.
206     */
207    @Handler
208    @SuppressWarnings("PMD.ConfusingTernary")
209    public void onVmDefChanged(VmDefChanged event, VmChannel channel)
210            throws ApiException, TemplateException, IOException {
211        // We're only interested in "spec" changes.
212        if (!event.specChanged()) {
213            return;
214        }
215
216        // Ownership relationships takes care of deletions
217        if (event.type() == K8sObserver.ResponseType.DELETED) {
218            logger.fine(
219                () -> "VM \"" + event.vmDefinition().name() + "\" deleted");
220            return;
221        }
222
223        // Create model for processing templates
224        Map<String, Object> model
225            = prepareModel(channel.client(), event.vmDefinition());
226        var configMap = cmReconciler.reconcile(model, channel);
227        model.put("cm", configMap);
228        dsReconciler.reconcile(event, model, channel);
229        // Manage (eventual) removal of stateful set.
230        stsReconciler.reconcile(event, model, channel);
231        pvcReconciler.reconcile(event, model, channel);
232        podReconciler.reconcile(event, model, channel);
233        lbReconciler.reconcile(event, model, channel);
234    }
235
236    /**
237     * Reset the VM by incrementing the reset count and doing a 
238     * partial reconcile (configmap only).
239     *
240     * @param event the event
241     * @param channel the channel
242     * @throws IOException 
243     * @throws ApiException 
244     * @throws TemplateException 
245     */
246    @Handler
247    public void onResetVm(ResetVm event, VmChannel channel)
248            throws ApiException, IOException, TemplateException {
249        var vmDef = channel.vmDefinition();
250        vmDef.extra("resetCount", vmDef.<Long> extra("resetCount") + 1);
251        Map<String, Object> model
252            = prepareModel(channel.client(), channel.vmDefinition());
253        cmReconciler.reconcile(model, channel);
254    }
255
256    @SuppressWarnings({ "PMD.CognitiveComplexity", "PMD.NPathComplexity" })
257    private Map<String, Object> prepareModel(K8sClient client,
258            VmDefinition vmDef) throws TemplateModelException, ApiException {
259        @SuppressWarnings("PMD.UseConcurrentHashMap")
260        Map<String, Object> model = new HashMap<>();
261        model.put("managerVersion",
262            Optional.ofNullable(Reconciler.class.getPackage()
263                .getImplementationVersion()).orElse("(Unknown)"));
264        model.put("cr", vmDef);
265        model.put("constants",
266            (TemplateHashModel) new DefaultObjectWrapperBuilder(
267                Configuration.VERSION_2_3_32)
268                    .build().getStaticModels()
269                    .get(Constants.class.getName()));
270        model.put("reconciler", config);
271
272        // Check if we have a display secret
273        ListOptions options = new ListOptions();
274        options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + ","
275            + "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + ","
276            + "app.kubernetes.io/instance=" + vmDef.name());
277        var dsStub = K8sV1SecretStub
278            .list(client, vmDef.namespace(), options)
279            .stream()
280            .findFirst();
281        if (dsStub.isPresent()) {
282            dsStub.get().model().ifPresent(m -> {
283                model.put("displaySecret", m.getMetadata().getName());
284            });
285        }
286
287        // Methods
288        model.put("parseQuantity", parseQuantityModel);
289        model.put("formatMemory", formatMemoryModel);
290        model.put("imageLocation", imgageLocationModel);
291        model.put("adjustCloudInitMeta", adjustCloudInitMetaModel);
292        model.put("toJson", toJsonModel);
293        return model;
294    }
295
296    private final TemplateMethodModelEx parseQuantityModel
297        = new TemplateMethodModelEx() {
298            @Override
299            @SuppressWarnings("PMD.PreserveStackTrace")
300            public Object exec(@SuppressWarnings("rawtypes") List arguments)
301                    throws TemplateModelException {
302                var arg = arguments.get(0);
303                if (arg instanceof SimpleNumber number) {
304                    return number.getAsNumber();
305                }
306                try {
307                    return Quantity.fromString(arg.toString()).getNumber();
308                } catch (NumberFormatException e) {
309                    throw new TemplateModelException("Cannot parse memory "
310                        + "specified as \"" + arg + "\": " + e.getMessage());
311                }
312            }
313        };
314
315    private final TemplateMethodModelEx formatMemoryModel
316        = new TemplateMethodModelEx() {
317            @Override
318            @SuppressWarnings("PMD.PreserveStackTrace")
319            public Object exec(@SuppressWarnings("rawtypes") List arguments)
320                    throws TemplateModelException {
321                var arg = arguments.get(0);
322                if (arg instanceof SimpleNumber number) {
323                    arg = number.getAsNumber();
324                }
325                BigInteger bigInt;
326                if (arg instanceof BigInteger value) {
327                    bigInt = value;
328                } else if (arg instanceof BigDecimal dec) {
329                    try {
330                        bigInt = dec.toBigIntegerExact();
331                    } catch (ArithmeticException e) {
332                        return arg;
333                    }
334                } else if (arg instanceof Integer value) {
335                    bigInt = BigInteger.valueOf(value);
336                } else if (arg instanceof Long value) {
337                    bigInt = BigInteger.valueOf(value);
338                } else {
339                    return arg;
340                }
341                return Convertions.formatMemory(bigInt);
342            }
343        };
344
345    private final TemplateMethodModelEx imgageLocationModel
346        = new TemplateMethodModelEx() {
347            @Override
348            @SuppressWarnings({ "PMD.PreserveStackTrace",
349                "PMD.AvoidLiteralsInIfCondition" })
350            public Object exec(@SuppressWarnings("rawtypes") List arguments)
351                    throws TemplateModelException {
352                var image = ((SimpleScalar) arguments.get(0)).getAsString();
353                if (image.isEmpty()) {
354                    return "";
355                }
356                try {
357                    var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH
358                        + "/").resolve(image);
359                    if ("file".equals(imageUri.getScheme())) {
360                        return imageUri.getPath();
361                    }
362                    return imageUri.toString();
363                } catch (URISyntaxException e) {
364                    logger.warning(() -> "Invalid CDROM image: " + image);
365                }
366                return image;
367            }
368        };
369
370    private final TemplateMethodModelEx adjustCloudInitMetaModel
371        = new TemplateMethodModelEx() {
372            @Override
373            @SuppressWarnings("PMD.PreserveStackTrace")
374            public Object exec(@SuppressWarnings("rawtypes") List arguments)
375                    throws TemplateModelException {
376                @SuppressWarnings("unchecked")
377                var res = (Map<String, Object>) DataPath
378                    .deepCopy(((AdapterTemplateModel) arguments.get(0))
379                        .getAdaptedObject(Object.class));
380                var metadata
381                    = (V1ObjectMeta) ((AdapterTemplateModel) arguments.get(1))
382                        .getAdaptedObject(Object.class);
383                if (!res.containsKey("instance-id")) {
384                    res.put("instance-id",
385                        Optional.ofNullable(metadata.getResourceVersion())
386                            .map(s -> "v" + s).orElse("v1"));
387                }
388                if (!res.containsKey("local-hostname")) {
389                    res.put("local-hostname", metadata.getName());
390                }
391                return res;
392            }
393        };
394
395    private final TemplateMethodModelEx toJsonModel
396        = new TemplateMethodModelEx() {
397            @Override
398            @SuppressWarnings("PMD.PreserveStackTrace")
399            public Object exec(@SuppressWarnings("rawtypes") List arguments)
400                    throws TemplateModelException {
401                try {
402                    return mapper.writeValueAsString(
403                        ((AdapterTemplateModel) arguments.get(0))
404                            .getAdaptedObject(Object.class));
405                } catch (JsonProcessingException e) {
406                    return "{}";
407                }
408            }
409        };
410}