Java

Distributed Tracing with Spring Boot, Infinispan, and Jaeger

Infinispan 14, or Datagrid 8.4, has the capability to implement a distributed tracing which will make distributed tracing easier and providing a better end-to-end view.

The concept of distributed tracing perhaps looks like this,

For this example, we are going to use Spring Boot 3 and OpenTelemetry for client side, and Jaeger all-in-one on for monitoring. So lets start with a simple pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.edw</groupId>
    <artifactId>spring-boot-infinispan-and-jaeger</artifactId>
    <version>1.0</version>

    <name>Spring Boot 3 with OpenTelemetry and Jaeger</name>
    <description>Spring Boot testing app with opentelemetry</description>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>3.0.5</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>

    <properties>
        <maven.compiler.source>17</maven.compiler.source>
        <maven.compiler.target>17</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

        <version.infinispan>14.0.2.Final</version.infinispan>
        <version.protostream>4.6.2.Final</version.protostream>
    </properties>

    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.infinispan</groupId>
                <artifactId>infinispan-bom</artifactId>
                <version>${version.infinispan}</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-actuator</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <!-- tracing -->
        <dependency>
            <groupId>io.micrometer</groupId>
            <artifactId>micrometer-tracing-bridge-otel</artifactId>
        </dependency>
        <dependency>
            <groupId>io.opentelemetry</groupId>
            <artifactId>opentelemetry-exporter-otlp</artifactId>
        </dependency>

        <!-- infinispan -->
        <dependency>
            <groupId>org.infinispan</groupId>
            <artifactId>infinispan-spring-boot-starter-remote</artifactId>
        </dependency>
        <dependency>
            <groupId>org.infinispan</groupId>
            <artifactId>infinispan-query</artifactId>
        </dependency>
        <dependency>
            <groupId>org.infinispan</groupId>
            <artifactId>infinispan-remote-query-client</artifactId>
        </dependency>
        <dependency>
            <groupId>org.infinispan</groupId>
            <artifactId>infinispan-api</artifactId>
        </dependency>

        <dependency>
            <groupId>org.infinispan</groupId>
            <artifactId>infinispan-spring-boot-starter-embedded</artifactId>
        </dependency>

        <dependency>
            <groupId>org.infinispan.protostream</groupId>
            <artifactId>protostream-processor</artifactId>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.infinispan</groupId>
            <artifactId>infinispan-client-hotrod</artifactId>
        </dependency>
        <dependency>
            <groupId>javax.transaction</groupId>
            <artifactId>jta</artifactId>
            <version>1.1</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

</project>

Next is some Java classes,

@SpringBootApplication
public class Main {
    public static void main(String[] args) {
        SpringApplication.run(Main.class, args);
    }
}

@RestController
public class IndexController {

    private static final Logger logger = LoggerFactory.getLogger(IndexController.class);

    @Autowired
    private IndexService indexService;

    @GetMapping("/")
    public ResponseEntity path1() {

        logger.info("request at / ");
        indexService.createData();
        return ResponseEntity.ok().build();
    }

}

@Service
public class IndexService {

    @Autowired
    private RemoteCacheManager cacheManager;

    @Autowired
    private Tracer tracer;

    private Logger logger = LoggerFactory.getLogger(this.getClass());

    public void createData() {
        logger.info("starting ====================");

        // starting span
        Span span = tracer.spanBuilder()
                .name("cache-add")
                .tag("cache", "balance")
                .kind(Span.Kind.CLIENT).start();

        final RemoteCache cache = cacheManager.getCache("balance");

        // generate cache key
        String key = UUID.randomUUID().toString();

        // put it
        cache.put(key, Math.random());

        // get it -- somehow this is not monitored in Jaeger
        Double value = Double.parseDouble((String)cache.get(key));

        // and remove it
        cache.remove(key);

        span.end();

        logger.info("done processing {} - {} ====================", key, value);
    }
}

@Configuration
public class OtlpConfiguration {

    @Bean
    public OtlpHttpSpanExporter otlpHttpSpanExporter() {
        return OtlpHttpSpanExporter.builder()
                .build();
    }

}

@Configuration
public class InfinispanConfiguration {
    @Bean
    public RemoteCacheManager remoteCacheManager() {
        return new RemoteCacheManager(
                new org.infinispan.client.hotrod.configuration.ConfigurationBuilder()
                        .addServers("127.0.0.1:11222")
                        .security().authentication().username("admin").password("password")
                        .clientIntelligence(ClientIntelligence.HASH_DISTRIBUTION_AWARE)
                        .marshaller(ProtoStreamMarshaller.class)
                        .build());
    }
}

server.port=8080
spring.application.name=app-testing

management.tracing.sampling.probability=1.0
management.otlp.metrics.export.url=http://localhost:4318/v1/traces
management.otlp.metrics.export.step=5s

logging.pattern.level=%5p [${spring.application.name:},%X{traceId:-},%X{spanId:-}]

Once we have all Java classes ready, we’ll continue with setting Jaeger server up. We’ll need

docker run -d \ 
        -e COLLECTOR_OTLP_ENABLED=true \ 
        -p 16686:16686 \ 
        -p 4317:4317 \
        -p 4318:4318 \ 
        jaegertracing/all-in-one:1.40

Next is setting up our Infinispan cache,

<distributed-cache name="balance" mode="SYNC" remote-timeout="30000" statistics="true">
    <encoding media-type="text/plain"/>
    <locking concurrency-level="1000" isolation="READ_COMMITTED" acquire-timeout="60000" striping="false"/>
    <transaction mode="NON_XA" auto-commit="true" stop-timeout="30000" locking="PESSIMISTIC" reaper-interval="30000" complete-timeout="30000" notifications="true"/>
    <state-transfer timeout="30000"/>
</distributed-cache>

And dont forget to put this Java runtime variables on our Infinispan,

JAVA_OPTS="$JAVA_OPTS -Dinfinispan.tracing.enabled=true -Dotel.service.name=infinispan-server
            -Dotel.exporter.otlp.endpoint=http://localhost:4317 -Dotel.metrics.exporter=none"

We can run this app, and see the result on our Jaeger dashboard.

And this is the detail traces within one same request,

Code for this article can be found on below Github link

https://github.com/edwin/spring-boot-infinispan-and-jaeger

Have fun.