report AbstractMethodError when use javaagent

[huange7]

when i run application with javaagent which is modified by me:

[otel.javaagent 2024-07-08 16:11:11:226 +0800] [main] DEBUG io.opentelemetry.javaagent.tooling.AgentInstaller$TransformLoggingListener - Failed to handle org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter for transformation on class loader jdk.internal.loader.ClassLoaders$AppClassLoader@4e0e2f2a
java.lang.AbstractMethodError: Receiver class io.opentelemetry.javaagent.tooling.bytebuddy.ExceptionHandlers$1 does not define or inherit an implementation of the resolved method 'abstract net.bytebuddy.implementation.bytecode.StackManipulation$Size apply(net.bytebuddy.jar.asm.MethodVisitor, net.bytebuddy.implementation.Implementation$Context)' of interface net.bytebuddy.implementation.bytecode.StackManipulation.
	at net.bytebuddy.asm.Advice$Dispatcher$SuppressionHandler$Suppressing$Bound.onEnd(Advice.java:8014)
	at net.bytebuddy.asm.Advice$Dispatcher$Inlining$CodeTranslationVisitor.visitEnd(Advice.java:10292)
	at net.bytebuddy.jar.asm.MethodVisitor.visitEnd(MethodVisitor.java:796)
	at net.bytebuddy.jar.asm.ClassReader.readMethod(ClassReader.java:1519)
	at net.bytebuddy.jar.asm.ClassReader.accept(ClassReader.java:745)
	at net.bytebuddy.jar.asm.ClassReader.accept(ClassReader.java:425)
	at net.bytebuddy.asm.Advice$Dispatcher$Inlining$Resolved$AdviceMethodInliner.apply(Advice.java:9230)
	at net.bytebuddy.asm.Advice$AdviceVisitor.onAfterExceptionTable(Advice.java:11524)
	at net.bytebuddy.utility.visitor.ExceptionTableSensitiveMethodVisitor.considerEndOfExceptionTable(ExceptionTableSensitiveMethodVisitor.java:49)
	at net.bytebuddy.utility.visitor.ExceptionTableSensitiveMethodVisitor.visitLabel(ExceptionTableSensitiveMethodVisitor.java:81)
	at net.bytebuddy.jar.asm.Label.accept(Label.java:369)
	at net.bytebuddy.jar.asm.ClassReader.readCode(ClassReader.java:2059)
	at net.bytebuddy.jar.asm.ClassReader.readMethod(ClassReader.java:1515)
	at net.bytebuddy.jar.asm.ClassReader.accept(ClassReader.java:745)
	at net.bytebuddy.jar.asm.ClassReader.accept(ClassReader.java:425)
	at net.bytebuddy.dynamic.scaffold.TypeWriter$Default$ForInlining.create(TypeWriter.java:4014)
	at net.bytebuddy.dynamic.scaffold.TypeWriter$Default.make(TypeWriter.java:2224)
	at net.bytebuddy.dynamic.DynamicType$Builder$AbstractBase$UsingTypeWriter.make(DynamicType.java:4062)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer.doTransform(AgentBuilder.java:12510)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer.transform(AgentBuilder.java:12445)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer.access$1800(AgentBuilder.java:12154)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer$Java9CapableVmDispatcher.run(AgentBuilder.java:12936)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer$Java9CapableVmDispatcher.run(AgentBuilder.java:12866)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:400)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer.doPrivileged(AgentBuilder.java)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer.transform(AgentBuilder.java:12388)
	at net.bytebuddy.agent.builder.AgentBuilder$Default$ExecutingTransformer$ByteBuddy$ModuleSupport.transform(Unknown Source)
	at java.instrument/sun.instrument.TransformerManager.transform(TransformerManager.java:188)
	at java.instrument/sun.instrument.InstrumentationImpl.transform(InstrumentationImpl.java:610)
	at java.base/java.lang.ClassLoader.defineClass1(Native Method)
	at java.base/java.lang.ClassLoader.defineClass(ClassLoader.java:1027)
	at java.base/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
	at java.base/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
	at java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:760)
	at java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
	at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
	at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
	at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:526)
	at java.base/java.lang.ClassLoader.defineClass1(Native Method)
	at java.base/java.lang.ClassLoader.defineClass(ClassLoader.java:1027)
	at java.base/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
	at java.base/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
	at java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:760)
	at java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
	at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
	at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
	at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:526)
	at java.base/java.lang.Class.getDeclaredMethods0(Native Method)
	at java.base/java.lang.Class.privateGetDeclaredMethods(Class.java:3578)
	at java.base/java.lang.Class.getDeclaredMethods(Class.java:2676)
	at org.springframework.util.ReflectionUtils.getDeclaredMethods(ReflectionUtils.java:465)
	at org.springframework.util.ReflectionUtils.doWithMethods(ReflectionUtils.java:360)
	at org.springframework.util.ReflectionUtils.getUniqueDeclaredMethods(ReflectionUtils.java:417)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.lambda$getTypeForFactoryMethod$1(AbstractAutowireCapableBeanFactory.java:748)
	at java.base/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1740)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.getTypeForFactoryMethod(AbstractAutowireCapableBeanFactory.java:747)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.determineTargetType(AbstractAutowireCapableBeanFactory.java:680)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.predictBeanType(AbstractAutowireCapableBeanFactory.java:651)
	at org.springframework.beans.factory.support.AbstractBeanFactory.isFactoryBean(AbstractBeanFactory.java:1632)
	at org.springframework.beans.factory.support.DefaultListableBeanFactory.doGetBeanNamesForType(DefaultListableBeanFactory.java:560)
	at org.springframework.beans.factory.support.DefaultListableBeanFactory.getBeanNamesForType(DefaultListableBeanFactory.java:532)
	at org.springframework.boot.autoconfigure.condition.OnBeanCondition.collectBeanNamesForType(OnBeanCondition.java:246)
	at org.springframework.boot.autoconfigure.condition.OnBeanCondition.getBeanNamesForType(OnBeanCondition.java:239)
	at org.springframework.boot.autoconfigure.condition.OnBeanCondition.getBeanNamesForType(OnBeanCondition.java:229)
	at org.springframework.boot.autoconfigure.condition.OnBeanCondition.getMatchingBeans(OnBeanCondition.java:182)
	at org.springframework.boot.autoconfigure.condition.OnBeanCondition.getMatchOutcome(OnBeanCondition.java:157)
	at org.springframework.boot.autoconfigure.condition.SpringBootCondition.matches(SpringBootCondition.java:47)
	at org.springframework.context.annotation.ConditionEvaluator.shouldSkip(ConditionEvaluator.java:108)
	at org.springframework.context.annotation.ConfigurationClassBeanDefinitionReader.loadBeanDefinitionsForBeanMethod(ConfigurationClassBeanDefinitionReader.java:183)
	at org.springframework.context.annotation.ConfigurationClassBeanDefinitionReader.loadBeanDefinitionsForConfigurationClass(ConfigurationClassBeanDefinitionReader.java:144)
	at org.springframework.context.annotation.ConfigurationClassBeanDefinitionReader.loadBeanDefinitions(ConfigurationClassBeanDefinitionReader.java:120)
	at org.springframework.context.annotation.ConfigurationClassPostProcessor.processConfigBeanDefinitions(ConfigurationClassPostProcessor.java:427)
	at org.springframework.context.annotation.ConfigurationClassPostProcessor.postProcessBeanDefinitionRegistry(ConfigurationClassPostProcessor.java:287)
	at org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanDefinitionRegistryPostProcessors(PostProcessorRegistrationDelegate.java:344)
	at org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanFactoryPostProcessors(PostProcessorRegistrationDelegate.java:115)
	at org.springframework.context.support.AbstractApplicationContext.invokeBeanFactoryPostProcessors(AbstractApplicationContext.java:771)
	at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:589)
	at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:146)
	at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:734)
	at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:436)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:312)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1306)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1295)
	at com.example.springboot3.Springboot3Application.main(Springboot3Application.java:10)

i have no idea which modification will cause such error, I am grateful for any reply.

[huange7]

my agent modify from 2.3.0

[laurit]

examine the output of ./gradlew -q :javaagent:dependencies

[huange7]

I found that it appears in jackson-databind. is this the reason?

+--- com.alibaba.nacos:nacos-client:2.2.3
|    |    |    |    |    +--- com.alibaba.nacos:nacos-auth-plugin:2.2.3
|    |    |    |    |    +--- com.alibaba.nacos:nacos-encryption-plugin:2.2.3
|    |    |    |    |    +--- commons-codec:commons-codec:1.15 -> 1.16.1
|    |    |    |    |    +--- com.fasterxml.jackson.core:jackson-core:2.12.6 -> 2.17.0
|    |    |    |    |    |    \--- com.fasterxml.jackson:jackson-bom:2.17.0
|    |    |    |    |    |         +--- com.fasterxml.jackson.core:jackson-core:2.17.0 (c)
|    |    |    |    |    |         +--- com.fasterxml.jackson.core:jackson-databind:2.17.0 (c)
|    |    |    |    |    |         +--- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.0 (c)
|    |    |    |    |    |         \--- com.fasterxml.jackson.core:jackson-annotations:2.17.0 (c)
|    |    |    |    |    +--- com.fasterxml.jackson.core:jackson-databind:2.12.7.1 -> 2.17.0
|    |    |    |    |    |    +--- com.fasterxml.jackson.core:jackson-annotations:2.17.0
|    |    |    |    |    |    |    \--- com.fasterxml.jackson:jackson-bom:2.17.0 (*)
|    |    |    |    |    |    +--- com.fasterxml.jackson.core:jackson-core:2.17.0 (*)
|    |    |    |    |    |    +--- net.bytebuddy:byte-buddy:1.14.9 -> 1.14.13
|    |    |    |    |    |    \--- com.fasterxml.jackson:jackson-bom:2.17.0 (*)
|    |    |    |    |    +--- org.apache.httpcomponents:httpasyncclient:4.1.3
|    |    |    |    |    |    +--- org.apache.httpcomponents:httpcore:4.4.6
|    |    |    |    |    |    +--- org.apache.httpcomponents:httpcore-nio:4.4.6
|    |    |    |    |    |    |    \--- org.apache.httpcomponents:httpcore:4.4.6
|    |    |    |    |    |    +--- org.apache.httpcomponents:httpclient:4.5.3
|    |    |    |    |    |    |    +--- org.apache.httpcomponents:httpcore:4.4.6
|    |    |    |    |    |    |    +--- commons-logging:commons-logging:1.2 -> 1.3.1
|    |    |    |    |    |    |    \--- commons-codec:commons-codec:1.9 -> 1.16.1
|    |    |    |    |    |    \--- commons-logging:commons-logging:1.2 -> 1.3.1
|    |    |    |    |    +--- io.prometheus:simpleclient:0.12.0
|    |    |    |    |    |    +--- io.prometheus:simpleclient_tracer_otel:0.12.0
|    |    |    |    |    |    |    \--- io.prometheus:simpleclient_tracer_common:0.12.0
|    |    |    |    |    |    \--- io.prometheus:simpleclient_tracer_otel_agent:0.12.0
|    |    |    |    |    |         \--- io.prometheus:simpleclient_tracer_common:0.12.0
|    |    |    |    |    \--- org.yaml:snakeyaml:2.0 -> 2.2

[huange7]

if it is, can i use exclude("net.bytebuddy", "byte-buddy") to resolve this problem?

[laurit]

yes, see

opentelemetry-java-instrumentation/javaagent-tooling/build.gradle.kts

Lines 27 to 30 in 2cb97fc

	implementation("io.opentelemetry:opentelemetry-sdk-extension-incubator") {
	// we use byte-buddy-dep
	exclude("net.bytebuddy", "byte-buddy")
	}

[huange7]

When I executed ResourceProviderTest.resourceProviderOrder(https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/javaagent/src/test/java/io/opentelemetry/javaagent/ResourceProviderTest.java#L17), I encountered a problem. The intention here is to ensure that the AWS resource provider is found after the regular container provider, but I found that their order is the same. How is the precedence order ensured?

[huange7]

@laurit Thank you for the help you've always provided. I'd like to ask for your assistance with this issue again, although it's not related to the current topic.
I encountered an error when running the ResourceProviderTest individually. It's clear that their loading order does not conform to what the test declares. I see that their order is the same, so how can I find the problem?

[huange7]

[huange7]

i decompiled the ExceptionHandlers$1 class in my agent jar.

package io.opentelemetry.javaagent.tooling.bytebuddy;

import net.bytebuddy.ClassFileVersion;
import net.bytebuddy.implementation.Implementation.Context;
import net.bytebuddy.implementation.bytecode.StackManipulation;
import net.bytebuddy.implementation.bytecode.StackManipulation.Size;
import org.objectweb.asm.Label;
import org.objectweb.asm.MethodVisitor;

class ExceptionHandlers$1 implements StackManipulation {
   private final Size size = new Size(-1, 2);

   public boolean isValid() {
      return true;
   }

   public Size apply(MethodVisitor mv, Context context) {
      String name = context.getInstrumentedType().getName();
      Label logStart = new Label();
      Label logEnd = new Label();
      Label eatException = new Label();
      Label handlerExit = new Label();
      boolean frames = context.getClassFileVersion().isAtLeast(ClassFileVersion.JAVA_V6);
      mv.visitTryCatchBlock(logStart, logEnd, eatException, "java/lang/Throwable");
      mv.visitLabel(logStart);
      mv.visitLdcInsn("Failed to handle exception in instrumentation for " + name);
      mv.visitInsn(95);
      mv.visitMethodInsn(184, ExceptionHandlers.access$000(), "logSuppressedError", "(Ljava/lang/String;Ljava/lang/Throwable;)V", false);
      mv.visitLabel(logEnd);
      mv.visitJumpInsn(167, handlerExit);
      mv.visitLabel(eatException);
      if (frames) {
         mv.visitFrame(4, 0, (Object[])null, 1, new Object[]{"java/lang/Throwable"});
      }

      mv.visitInsn(87);
      mv.visitLabel(handlerExit);
      if (frames) {
         mv.visitFrame(3, 0, (Object[])null, 0, (Object[])null);
         mv.visitInsn(0);
      }

      return this.size;
   }
}