Add binding/dom codec class loader support 98/81698/5
authorRobert Varga <robert.varga@pantheon.tech>
Thu, 18 Apr 2019 11:07:22 +0000 (13:07 +0200)
committerRobert Varga <robert.varga@pantheon.tech>
Thu, 18 Apr 2019 15:13:11 +0000 (17:13 +0200)
Our current codec implementation relies on creating classes in
classloaders holding compile-time-generated code, effectively
polluting them, in two distinct ways.

The first, less intrusive, is the instantiation of dynamic
proxies, which support LazyDataObject and ForeignOpaqueObject
instances. Since this is done via JVM interfaces, the JVM
takes care of reusing these classes, but still they cannot
be removed when the binding-dom-codec is unloaded -- fortunately
their nature of being derived from yang-binding and compile-time
generated interfaces allows them to be reused.

The second, more intrusive, is the instantiation of dynamic
streamers to support emitting NormalizedNode structures from
DataObject instances. While there are some provisions to reuse
them, they are inherently tied to how BindingRuntimeContext
interprets generated classes -- if the underlying SchemaContext
ends up interpreting them incompatibly (for example, by leaf
changing to anyxml), we end up between a rock and a hard place --
the classes do not match NormalizedNode world and they cannot
be reloaded to rectify the problem.

This patch adds the infrastructure for separating out compile-time
and run-time worlds by introducing StaticClassPool and
CodecClassLoader. StaticClassPool is used to bind Javassist code
generator to the classloader into which binding-dom-codec is loaded.

CodecClassLoader allows effective bridging of StaticClassPool and
whatever classloader is loading compile-time-generated classes,
and provides a place where new classes can be defined without
polluting either classloader.

Change-Id: I963c7bc6eefd5dd77a80e06442896265116664c0
JIRA: MDSAL-442
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>
binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/CodecClassLoader.java [new file with mode: 0644]
binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/LeafCodecClassLoader.java [new file with mode: 0644]
binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/RootCodecClassLoader.java [new file with mode: 0644]
binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/StaticClassPool.java [new file with mode: 0644]
binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/package-info.java [new file with mode: 0644]

diff --git a/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/CodecClassLoader.java b/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/CodecClassLoader.java
new file mode 100644 (file)
index 0000000..22f7396
--- /dev/null
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.mdsal.binding.dom.codec.loader;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Verify.verify;
+import static com.google.common.base.Verify.verifyNotNull;
+
+import com.google.common.annotations.Beta;
+import com.google.common.base.Strings;
+import java.io.IOException;
+import javassist.CannotCompileException;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.LoaderClassPath;
+import javassist.NotFoundException;
+import org.eclipse.jdt.annotation.NonNull;
+import org.opendaylight.mdsal.binding.spec.reflect.BindingReflections;
+
+/**
+ * A ClassLoader hosting types generated for a particular type. A root instance is attached to a
+ * BindingCodecContext instance, so any generated classes from it can be garbage-collected when the context
+ * is destroyed, as well as to prevent two contexts trampling over each other.
+ *
+ * <p>
+ * It semantically combines two class loaders: the class loader in which this class is loaded and the class loader in
+ * which a target Binding interface/class is loaded. This inherently supports multi-classloader environments -- the root
+ * instance has visibility only into codec classes and for each classloader we encounter when presented with a binding
+ * class we create a leaf instance and cache it in the root instance. Leaf instances are using the root loader as their
+ * parent, but consult the binding class's class loader if the root loader fails to load a particular class.
+ *
+ * <p>In single-classloader environments, obviously, the root loader can load all binding classes, and hence no leaf
+ * loader is created.
+ *
+ * <p>
+ * Each {@link CodecClassLoader} has a {@link ClassPool} attached to it and can perform operations on it. Leaf loaders
+ * specify the root loader's ClassPool as their parent, but are configured to lookup classes first in themselves.
+ *
+ * @author Robert Varga
+ */
+@Beta
+public abstract class CodecClassLoader extends ClassLoader {
+    /**
+     * A customizer allowing a generated class to be modified before it is loader.
+     */
+    @FunctionalInterface
+    public interface Customizer {
+        /**
+         * Customize a generated class before it is instantiated in the loader.
+         *
+         * @param loader CodecClassLoader which will hold the class. It can be used to lookup/instantiate other classes
+         * @param bindingClass Binding class for which the customized class is being generated
+         * @param generated The class being generated
+         * @throws CannotCompileException if the customizer cannot perform partial compilation
+         * @throws NotFoundException if the customizer cannot find a required class
+         * @throws IOException if the customizer cannot perform partial loading
+         */
+        void customize(@NonNull CodecClassLoader loader, @NonNull CtClass bindingClass, @NonNull CtClass generated)
+                throws CannotCompileException, NotFoundException, IOException;
+    }
+
+    static {
+        verify(ClassLoader.registerAsParallelCapable());
+    }
+
+    private final ClassPool classPool;
+
+    private CodecClassLoader(final ClassLoader parentLoader, final ClassPool parentPool) {
+        super(parentLoader);
+        this.classPool = new ClassPool(parentPool);
+        this.classPool.childFirstLookup = true;
+        this.classPool.appendClassPath(new LoaderClassPath(this));
+    }
+
+    CodecClassLoader() {
+        this(StaticClassPool.LOADER, StaticClassPool.POOL);
+    }
+
+    CodecClassLoader(final CodecClassLoader parent) {
+        this(parent, parent.classPool);
+    }
+
+    /**
+     * Turn a Class instance into a CtClass for referencing it in code generation. This method supports both
+     * generated- and non-generated classes.
+     *
+     * @param clazz Class to be looked up.
+     * @return A CtClass instance
+     * @throws NotFoundException if the class cannot be found
+     * @throws NullPointerException if {@code clazz} is null
+     */
+    public final @NonNull CtClass findClass(final @NonNull Class<?> clazz) throws NotFoundException {
+        return BindingReflections.isBindingClass(clazz) ? findClassLoader(clazz).getLocalFrozen(clazz.getName())
+                : StaticClassPool.findClass(clazz);
+    }
+
+    /**
+     * Create a new class by subclassing specified class and running a customizer on it. The name of the target class
+     * is formed through concatenation of the name of a {@code bindingInterface} and specified {@code suffix}
+     *
+     * @param superClass Superclass from which to derive
+     * @param bindingInterface Binding compile-time-generated interface
+     * @param suffix Suffix to use
+     * @param customizer Customizer to use to process the class
+     * @return A generated class object
+     * @throws CannotCompileException if the resulting generated class cannot be compiled or customized
+     * @throws NotFoundException if the binding interface cannot be found or the generated class cannot be customized
+     * @throws IOException if the generated class cannot be turned into bytecode or the generator fails with IOException
+     * @throws NullPointerException if any argument is null
+     */
+    public final Class<?> generateSubclass(final CtClass superClass, final Class<?> bindingInterface,
+            final String suffix, final Customizer customizer) throws CannotCompileException, IOException,
+                NotFoundException {
+        return findClassLoader(bindingInterface).doGenerateSubclass(superClass, bindingInterface, suffix, customizer);
+    }
+
+    final @NonNull CtClass getLocalFrozen(final String name) throws NotFoundException {
+        synchronized (getClassLoadingLock(name)) {
+            final CtClass result = classPool.get(name);
+            result.freeze();
+            return result;
+        }
+    }
+
+    abstract @NonNull CodecClassLoader findClassLoader(Class<?> bindingClass);
+
+    private Class<?> doGenerateSubclass(final CtClass superClass, final Class<?> bindingInterface, final String suffix,
+            final Customizer customizer) throws CannotCompileException, IOException, NotFoundException {
+        checkArgument(!superClass.isInterface(), "%s must not be an interface", superClass);
+        checkArgument(bindingInterface.isInterface(), "%s is not an interface", bindingInterface);
+        checkArgument(!Strings.isNullOrEmpty(suffix));
+
+        final String bindingName = bindingInterface.getName();
+        final String fqn = bindingName + "$$$" + suffix;
+        synchronized (getClassLoadingLock(fqn)) {
+            // Attempt to find a loaded class
+            final Class<?> loaded = findLoadedClass(fqn);
+            if (loaded != null) {
+                return loaded;
+            }
+
+            // Get the interface
+            final CtClass bindingCt = getLocalFrozen(bindingName);
+            try {
+                final byte[] byteCode;
+                final CtClass generated = verifyNotNull(classPool.makeClass(fqn, superClass));
+                try {
+                    customizer.customize(this, bindingCt, generated);
+                    final String ctName = generated.getName();
+                    verify(fqn.equals(ctName), "Target class is %s returned result is %s", fqn, ctName);
+                    byteCode = generated.toBytecode();
+                } finally {
+                    // Always detach the result, as we will never use it again
+                    generated.detach();
+                }
+
+                final Class<?> newClass = defineClass(fqn, byteCode, 0, byteCode.length);
+                resolveClass(newClass);
+                return newClass;
+            } finally {
+                // Binding interfaces are used only a few times, hence it does not make sense to cache them in the class
+                // pool.
+                // TODO: this hinders caching, hence we should re-think this
+                bindingCt.detach();
+            }
+        }
+    }
+}
diff --git a/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/LeafCodecClassLoader.java b/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/LeafCodecClassLoader.java
new file mode 100644 (file)
index 0000000..fdf2903
--- /dev/null
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.mdsal.binding.dom.codec.loader;
+
+import static com.google.common.base.Verify.verify;
+import static java.util.Objects.requireNonNull;
+
+import org.eclipse.jdt.annotation.NonNull;
+
+// A leaf class loader, binding together a root class loader and some other class loader
+final class LeafCodecClassLoader extends CodecClassLoader {
+    static {
+        verify(registerAsParallelCapable());
+    }
+
+    private final @NonNull ClassLoader target;
+    private final @NonNull RootCodecClassLoader root;
+
+    LeafCodecClassLoader(final RootCodecClassLoader root, final ClassLoader target) {
+        super(root);
+        this.root = requireNonNull(root);
+        this.target = requireNonNull(target);
+    }
+
+    @Override
+    protected Class<?> findClass(final String name) throws ClassNotFoundException {
+        return target.loadClass(name);
+    }
+
+    @Override
+    CodecClassLoader findClassLoader(final Class<?> bindingClass) {
+        final ClassLoader bindingTarget = bindingClass.getClassLoader();
+        return target.equals(bindingTarget) ? this : root.findClassLoader(bindingClass);
+    }
+}
diff --git a/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/RootCodecClassLoader.java b/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/RootCodecClassLoader.java
new file mode 100644 (file)
index 0000000..225941d
--- /dev/null
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.mdsal.binding.dom.codec.loader;
+
+import static com.google.common.base.Verify.verify;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// A root codec classloader, binding only whatever is available StaticClassPool
+final class RootCodecClassLoader extends CodecClassLoader {
+    private static final Logger LOG = LoggerFactory.getLogger(RootCodecClassLoader.class);
+
+    static {
+        verify(registerAsParallelCapable());
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static final AtomicReferenceFieldUpdater<RootCodecClassLoader, ImmutableMap> LOADERS_UPDATER =
+            AtomicReferenceFieldUpdater.newUpdater(RootCodecClassLoader.class, ImmutableMap.class, "loaders");
+
+    private volatile ImmutableMap<ClassLoader, CodecClassLoader> loaders = ImmutableMap.of();
+
+    RootCodecClassLoader() {
+        super();
+    }
+
+    @Override
+    CodecClassLoader findClassLoader(final Class<?> bindingClass) {
+        final ClassLoader target = bindingClass.getClassLoader();
+        if (target == null) {
+            // No class loader associated ... well, let's use root then
+            return this;
+        }
+
+        // Cache for update
+        ImmutableMap<ClassLoader, CodecClassLoader> local = loaders;
+        final CodecClassLoader known = local.get(target);
+        if (known != null) {
+            return known;
+        }
+
+        // Alright, we need to determine if the class is accessible through our hierarchy (in which case we use
+        // ourselves) or we need to create a new Leaf.
+        final CodecClassLoader found;
+        if (!isOurClass(bindingClass)) {
+            StaticClassPool.verifyStaticLinkage(target);
+            found = AccessController.doPrivileged(
+                (PrivilegedAction<CodecClassLoader>)() -> new LeafCodecClassLoader(this, target));
+        } else {
+            found = this;
+        }
+
+        // Now make sure we cache this result
+        while (true) {
+            final Builder<ClassLoader, CodecClassLoader> builder = ImmutableMap.builderWithExpectedSize(
+                local.size() + 1);
+            builder.putAll(local);
+            builder.put(target, found);
+
+            if (LOADERS_UPDATER.compareAndSet(this, local, builder.build())) {
+                return found;
+            }
+
+            local = loaders;
+            final CodecClassLoader recheck = local.get(target);
+            if (recheck != null) {
+                return recheck;
+            }
+        }
+    }
+
+    private boolean isOurClass(final Class<?> bindingClass) {
+        final Class<?> ourClass;
+        try {
+            ourClass = loadClass(bindingClass.getName(), false);
+        } catch (ClassNotFoundException e) {
+            LOG.debug("Failed to load {}", bindingClass, e);
+            return false;
+        }
+        return bindingClass.equals(ourClass);
+    }
+}
diff --git a/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/StaticClassPool.java b/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/StaticClassPool.java
new file mode 100644 (file)
index 0000000..a2b7ac6
--- /dev/null
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.mdsal.binding.dom.codec.loader;
+
+import static com.google.common.base.Verify.verify;
+import static com.google.common.base.Verify.verifyNotNull;
+
+import com.google.common.annotations.Beta;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.LoaderClassPath;
+import javassist.NotFoundException;
+import org.eclipse.jdt.annotation.NonNull;
+import org.opendaylight.yangtools.yang.binding.DataContainer;
+
+/**
+ * Static class pool, bound to the class loader of binding-dom-codec. It can be used to acquire CtClass instances that
+ * reside within the binding-dom-codec artifact or any of its direct mandatory dependencies. It can also instantiate
+ * {@link CodecClassLoader} instances for use with code generation.
+ *
+ * @author Robert Varga
+ */
+@Beta
+public final class StaticClassPool {
+    static final ClassLoader LOADER = verifyNotNull(StaticClassPool.class.getClassLoader());
+    static final ClassPool POOL;
+
+    static {
+        final ClassPool pool = new ClassPool();
+        pool.appendClassPath(new LoaderClassPath(LOADER));
+        POOL = pool;
+    }
+
+    private StaticClassPool() {
+        // Utility class
+    }
+
+    /**
+     * Instantiate a new CodecClassLoader.
+     *
+     * @return A new CodecClassLoader.
+     */
+    public static @NonNull CodecClassLoader createLoader() {
+        return AccessController.doPrivileged((PrivilegedAction<CodecClassLoader>)() -> new RootCodecClassLoader());
+    }
+
+    /**
+     * Resolve a binding-dom-codec class to its {@link CtClass} counterpart.
+     *
+     * @param clazz Class to resolve
+     * @return A CtClass instance
+     * @throws IllegalStateException if the class cannot be resolved
+     * @throws NullPointerException if {@code clazz} is null
+     */
+    public static synchronized @NonNull CtClass findClass(final @NonNull Class<?> clazz) {
+        final CtClass ret;
+        try {
+            ret = POOL.get(clazz.getName());
+        } catch (NotFoundException e) {
+            throw new IllegalStateException("Failed to find " + clazz, e);
+        }
+        ret.freeze();
+        return ret;
+    }
+
+    // Sanity check: target has to resolve yang-binding contents to the same class, otherwise we are in a pickle
+    static void verifyStaticLinkage(final ClassLoader candidate) {
+        final Class<?> targetClazz;
+        try {
+            targetClazz = candidate.loadClass(DataContainer.class.getName());
+        } catch (ClassNotFoundException e) {
+            throw new IllegalStateException("ClassLoader " + candidate + " cannot load " + DataContainer.class, e);
+        }
+        verify(DataContainer.class.equals(targetClazz),
+            "Class mismatch on DataContainer. Ours is from %s, target %s has %s from %s",
+            DataContainer.class.getClassLoader(), candidate, targetClazz, targetClazz.getClassLoader());
+    }
+}
diff --git a/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/package-info.java b/binding/mdsal-binding-dom-codec/src/main/java/org/opendaylight/mdsal/binding/dom/codec/loader/package-info.java
new file mode 100644 (file)
index 0000000..d66ec12
--- /dev/null
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+/**
+ * {@link java.lang.ClassLoader} support for Binding/DOM codec translation code generators. This package provides two
+ * core classes:
+ * <ul>
+ * <li>{@link org.opendaylight.mdsal.binding.dom.codec.loader.StaticClassPool}, which is allows lookup of classes within
+ *     Binding/DOM codec for the purpose of referencing them within code generators.</li>
+ * <li>{@link org.opendaylight.mdsal.binding.dom.codec.loader.CodecClassLoader}, which allows lookup of
+ *     compile-time-generated Binding classes for the purpose of referencing them within code generators and which
+ *     serves as the ClassLoader holding runtime-generated codecs.
+ * </li>
+ * </ul>
+ */
+package org.opendaylight.mdsal.binding.dom.codec.loader;
\ No newline at end of file