Java

BPS (Badan Pusat Statistik) is a non-departmental government institute of Indonesia that is responsible for conducting statistical surveys. On its website, we can see that there are a lot of data available, especially regarding to spatial and regional such as number of provinces, zipcodes, cities, and others.

One BPS website which is contains interesting data is https://sig.bps.go.id/, an on this sample im trying to crawl and read the zipcode data from it.

Lets start with pom file,

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.edw</groupId>
    <artifactId>ScrappingBPS</artifactId>
    <version>1.0</version>

    <properties>
        <java.version>11</java.version>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>okhttp</artifactId>
            <version>4.9.3</version>
        </dependency>
        <dependency>
            <groupId>org.mybatis</groupId>
            <artifactId>mybatis</artifactId>
            <version>3.5.5</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.25</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.13.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-api</artifactId>
            <version>2.15.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.15.0</version>
        </dependency>

        <dependency>
            <groupId>com.lmax</groupId>
            <artifactId>disruptor</artifactId>
            <version>3.4.4</version>
        </dependency>

    </dependencies>
</project>

Since im using MyBatis, so lets create a SessionFactory class.

package com.edw.config;

import com.edw.mapper.KodeposMapper;
import com.edw.mapper.RegionsMapper;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;

import java.io.IOException;
import java.io.Reader;

public class MyBatisSqlSessionFactory {
    private static final SqlSessionFactory FACTORY;

    static {
        try {
            Reader reader = Resources.getResourceAsReader("configuration.xml");
            FACTORY = new SqlSessionFactoryBuilder().build(reader);
            FACTORY.getConfiguration().addMapper(KodeposMapper.class);
            
        } catch (IOException e) {
            throw new RuntimeException("Fatal Error. Cause: " + e, e);
        }
    }

    public static SqlSessionFactory getSqlSessionFactory() {
        return FACTORY;
    }
}

And a bean for table representation, and a mapper interface.

package com.edw.bean;

import java.io.Serializable;

public class Kodepos implements Serializable {
    private Long id;
    private String kelurahan;
    private String kecamatan;
    private String kabupaten;
    private String provinsi;
    private String kodepos;

    public Kodepos() {
    }

    // other setter and getter
}

package com.edw.mapper;

import com.edw.bean.Kodepos;
import org.apache.ibatis.annotations.Insert;

public interface KodeposMapper {

    @Insert("INSERT INTO `db_kodepos`.`tbl_kodepos` " +
            "(`kelurahan`, `kecamatan`, `kabupaten`, `provinsi`, `kodepos`) " +
            "VALUES (#{kelurahan}, #{kecamatan}, #{kabupaten}, #{provinsi}, #{kodepos})")
    Integer insert(Kodepos kodepos);
}

And last is our main class, im using okhttp to do http request and jackson for parsing json to java object

package com.edw;

import com.edw.bean.Kodepos;
import com.edw.config.MyBatisSqlSessionFactory;
import com.edw.mapper.KodeposMapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.ibatis.session.SqlSession;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;

public class KodeposScrapping {

    private SqlSession sqlSession = null;
    private KodeposMapper kodeposMapper  = null;

    private static final String BASE_URL = "https://sig.bps.go.id";

    private Logger logger = LogManager.getLogger(KodeposScrapping.class);

    public KodeposScrapping() {
    }

    public static void main(String[] args) throws IOException {
        KodeposScrapping kodeposScrapping = new KodeposScrapping();
        kodeposScrapping.doScrapping();
    }

    private void doScrapping() throws IOException {
        getProvinsis();
    }

    /**
     *  <pre> curl "https://sig.bps.go.id/rest-drop-down/getwilayah" </pre>
     */
    private void getProvinsis() throws IOException {
        Request request = new Request.Builder()
                .url(BASE_URL + "/rest-drop-down/getwilayah")
                .get()
                .build();

        List<HashMap> hashMaps = doHttpCall(request);

        for (HashMap hashMap : hashMaps) {
            Kodepos kodepos = new Kodepos();
            kodepos.setProvinsi(hashMap.get("nama").toString());

            logger.info("start processing {}", hashMap.get("nama").toString());

            // get Kabupatens from Provinsi
            getKabupatens(hashMap.get("kode").toString(), kodepos);

            logger.info("done processing {}", hashMap.get("nama").toString());
        }
    }

    /**
     *  <pre> curl "https://sig.bps.go.id/rest-drop-down/getwilayah?level=kabupaten&parent=11" </pre>
     */
    private void getKabupatens(String parent, Kodepos kodepos) throws IOException {
        Request request = new Request.Builder()
                .url(BASE_URL + "/rest-drop-down/getwilayah?level=kabupaten&parent="+parent)
                .get()
                .build();

        List<HashMap> hashMaps = doHttpCall(request);

        for (HashMap hashMap : hashMaps) {
            kodepos.setKabupaten(hashMap.get("nama").toString());

            // get Kecamatans from Kabupaten
            getKecamatans(hashMap.get("kode").toString(), kodepos);
        }
    }

    /**
     *  <pre> curl "https://sig.bps.go.id/rest-drop-down/getwilayah?level=kecamatan&parent=1101" </pre>
     */
    private void getKecamatans(String parent, Kodepos kodepos) throws IOException {
        Request request = new Request.Builder()
                .url(BASE_URL + "/rest-drop-down/getwilayah?level=kecamatan&parent="+parent)
                .get()
                .build();

        List<HashMap> hashMaps = Collections.synchronizedList(doHttpCall(request));

        for (HashMap hashMap : hashMaps) {
            kodepos.setKecamatan(hashMap.get("nama").toString());
            getKelurahans(hashMap.get("kode").toString(), kodepos);
        }
    }

    /**
     *  <pre> curl "https://sig.bps.go.id/rest-bridging-pos/getwilayah?level=desa&parent=1101050" </pre>
     */
    private void getKelurahans(String parent, Kodepos kodepos) throws IOException {
        Request request = new Request.Builder()
                .url(BASE_URL + "/rest-bridging-pos/getwilayah?level=desa&parent="+parent)
                .get()
                .build();

        List<HashMap> hashMaps = Collections.synchronizedList(doHttpCall(request));

        for (final HashMap hashMap : hashMaps) {
            kodepos.setKelurahan(hashMap.get("nama_bps").toString());
            kodepos.setKodepos(hashMap.get("kode_pos").toString());

            insert(kodepos);
        }
    }

    private void insert(Kodepos kodepos) {
        try {
            sqlSession = MyBatisSqlSessionFactory.getSqlSessionFactory().openSession(true);
            kodeposMapper = sqlSession.getMapper(KodeposMapper.class);

            kodeposMapper.insert(kodepos);
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
        } finally {
            if (sqlSession != null) {
                sqlSession.close();
            }
        }
    }

    private List<HashMap> doHttpCall(Request request) throws IOException {
        Call call = new OkHttpClient().newBuilder()
                .retryOnConnectionFailure(true)
                .connectTimeout(300, TimeUnit.SECONDS)
                .readTimeout(300, TimeUnit.SECONDS)
                .writeTimeout(300, TimeUnit.SECONDS).build().newCall(request);
        Response response = call.execute();

        ObjectMapper objectMapper = new ObjectMapper();
        List<HashMap> hashMaps = objectMapper.readValue(response.body().string(), List.class);

        response.close();
        return hashMaps;
    }
}

Code can be found in my github repository,

https://github.com/edwin/bps-data-scrapper