application.conf

{
  # Spark relation config
  spark: {
    app: {
      name: Nebula Exchange 2.0
    }

    master:local

    driver: {
      cores: 1
      maxResultSize: 1G
    }

    executor: {
        memory:1G
    }

    cores:{
      max: 16
    }
  }

  # if the hive is hive-on-spark with derby mode， you can ignore this hive configure
  # get the config values from file $HIVE_HOME/conf/hive-site.xml or hive-default.xml

  #  hive: {
  #    warehouse: "hdfs://NAMENODE_IP:9000/apps/svr/hive-xxx/warehouse/"
  #    connectionURL: "jdbc:mysql://your_ip:3306/hive_spark?characterEncoding=UTF-8"
  #    connectionDriverName: "com.mysql.jdbc.Driver"
  #    connectionUserName: "user"
  #    connectionPassword: "password"
  #  }


  # Nebula Graph relation config
  nebula: {
    address:{
      graph:["127.0.0.1:9669"]
      meta:["127.0.0.1:9559"]
    }
    user: root
    pswd: nebula
    space: test

    # parameters for SST import, not required
    path:{
        local:"/tmp"
        remote:"/sst"
        hdfs.namenode: "hdfs://name_node:9000"
    }

    # nebula client connection parameters
    connection {
      # socket connect & execute timeout, unit: millisecond
      timeout: 30000
    }

    error: {
      # max number of failures, if the number of failures is bigger than max, then exit the application.
      max: 32
      # failed import job will be recorded in output path
      output: /tmp/errors
    }

    # use google's RateLimiter to limit the requests send to NebulaGraph
    rate: {
      # the stable throughput of RateLimiter
      limit: 1024
      # Acquires a permit from RateLimiter, unit: MILLISECONDS
      # if it can't be obtained within the specified timeout, then give up the request.
      timeout: 1000
    }
  }

  # Processing tags
  # There are tag config examples for different dataSources.
  tags: [

    # HDFS parquet
    # Import mode is client, just change type.sink to sst if you want to use sst import mode.
    {
      name: tag-name-0
      type: {
        source: parquet
        sink: client
      }
      path: hdfs tag path 0
      fields: [parquet-field-0, parquet-field-1, parquet-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      vertex: {
        field:parquet-field-0
        #policy:hash
      }
      batch: 256
      partition: 32
    }

    # HDFS csv
    # Import mode is sst, just change type.sink to client if you want to use client import mode.
    {
      name: tag-name-1
      type: {
        source: csv
        sink: sst
      }
      path: hdfs tag path 2
      # if your csv file has no header, then use _c0,_c1,_c2,.. to indicate fields
      fields: [csv-field-0, csv-field-1, csv-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      vertex: {
        field:csv-field-0
      }
      separator: ","
      header: true
      batch: 256
      partition: 32
    }

    # HDFS json
    {
      name: tag-name-2
      type: {
        source: json
        sink: client
      }
      path: hdfs vertex path 3
      fields: [json-field-0, json-field-1, json-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      vertex: {
        field: json-field-0
        #policy: hash
      }
      batch: 256
      partition: 32
    }

    # Hive
    {
      name: tag-name-3
      type: {
        source: hive
        sink: client
      }
      exec: "select hive-field0, hive-field1, hive-field2 from database.table"
      fields: [hive-field-0, hive-field-1, hive-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      vertex: {
        field: hive-field-0
        # policy: "hash"
      }
      batch: 256
      partition: 32
    }

    # neo4j
    {
      name: tag-name-4
      type: {
        source: neo4j
        sink: client
      }
      server: "bolt://127.0.0.1:7687"
      user: neo4j
      password: neo4j
      exec: "match (n:label) return n.neo4j-field-0 as neo4j-field-0, n.neo4j-field-1 as neo4j-field-1 order by (n.neo4j-field-0)"
      fields: [neo4j-field-0, neo4j-field-1]
      nebula.fields: [nebula-field-0, nebula-field-1]
      vertex: {
        field:neo4j-field-0
        # policy:hash
      }
      partition: 10
      batch: 1000
      check_point_path: /tmp/test
   }

    # HBase
    # if fields or vertex contains rowkey, please configure it as "rowkey".
    {
      name: tag-name-5
      type: {
        source: hbase
        sink: client
      }
      host:127.0.0.1
      port:2181
      table:hbase-table
      columnFamily:hbase-table-cloumnfamily
      fields: [hbase-column-0, hbase-column-1]
      nebula.fields: [nebula-field-0, nebula-field-1]
      vertex: {
        field:rowkey
      }
      partition: 10
      batch: 1000
    }

    # Pulsar
    {
      name: tag-name-6
      type: {
        source: pulsar
        sink: client
      }
      service: "pulsar://localhost:6650"
      admin: "http://localhost:8081"
      options: {
        # choose one of "topic", "topics", "topicsPattern"
        topics: "topic1,topic2"
      }
      fields: [pulsar-field-0, pulsar-field-1, pulsar-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      vertex: {
        field:pulsar-field-0
      }
      partition: 10
      batch: 1000
      interval.seconds: 10
    }

    # KAFKA
    {
      name: tag-name-7
      type: {
        source: kafka
        sink: client
      }
      service: "kafka.service.address"
      topic: "topic-name"
      fields: [kafka-field-0, kafka-field-1, kafka-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      vertex: {
        field: kafka-field-0
      }
      partition: 10
      batch: 10
      interval.seconds: 10
    }

    # MaxCompute
    {
      name: tag-name-8
      type:{
        source:maxcompute
        sink:client
      }
      table:table
      project:project
      odpsUrl:"http://service.cn-hangzhou.maxcompute.aliyun.com/api"
      tunnelUrl:"http://dt.cn-hangzhou.maxcompute.aliyun.com"
      accessKeyId:xxx
      accessKeySecret:xxx
      partitionSpec:"dt='partition1'"
      # maxcompute sql sentence only uses table name. make sure that table name is the same with {table}'s value'.
      sentence:"select id, maxcompute-field-0, maxcompute-field-1, maxcompute-field-2 from table where id < 10"
      fields:[maxcompute-field-0, maxcompute-field-1]
      nebula.fields:[nebula-field-0, nebula-field-1]
      vertex:{
        field: maxcompute-field-2
      }
      partition:10
      batch:10
    }

    # ClickHouse
   {
      name: tag-name-8
      type: {
        source: clickhouse
        sink: client
      }
      url:"jdbc:clickhouse://127.0.0.1:8123/database"
      user:"user"
      password:"clickhouse"
      numPartition:"5"
      sentence:"select * from table"
      fields: [clickhouse-field-0, clickhouse-field-1, clickhouse-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      vertex: {
        field:clickhouse-field-0
        #policy:hash
      }
      batch: 256
      partition: 32
    }
  ]

  # Processing edges
  # There are edge config examples for different dataSources.
  edges: [
     # HDFS parquet
     # Import mode is client, just change type.sink to sst if you want to use sst import mode.
    {
      name: edge-name-0
      type: {
         source: parquet
         sink: client
      }
      path: hdfs edge path 0
      fields: [parquet-field-0, parquet-field-1, parquet-field-2]
      nebula.fields: [nebula-field-0 nebula-field-1 nebula-field-2]
      source: {
        field:parquet-field-0
        #policy:hash
      }
      target: {
        field:parquet-field-1
        #policy:hash
      }
      batch: 256
      partition: 32
    }

    # HDFS csv
    {
      name: edge-name-1
      type: {
        source: csv
        sink: client
      }
      path: hdfs edge path 1
      fields: [csv-field-0, csv-field-1, csv-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      source: {
        field: csv-field-0
        #policy: hash
      }
      target: {
        field: csv-field-1
      }
      ranking: csv-field-2
      separator: ","
      header: true
      batch: 256
      partition: 32
    }

    # HDFS json
    {
      name: edge-name-2
      type: {
        source: json
        sink: client
      }
      path: hdfs edge path 2
      fields: [json-field-0, json-field-1, json-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      source: {
        field: json-field-0
        #policy: hash
      }
      target: {
        field: json-field-1
      }
      ranking: json-field-2
      batch: 256
      partition: 32
    }

    # Hive
    {
      name: edge-name-2
      type: {
        source: hive
        sink: client
      }
      exec: "select hive-field0, hive-field1, hive-field2 from database.table"
      fields: [ hive-field-0, hive-field-1, hive-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      source: hive-field-0
      target: hive-field-1
      batch: 256
      partition: 32
    }

    # Neo4j
    {
      name: edge-name-3
      type: {
        source: neo4j
        sink: client
      }
      server: "bolt://127.0.0.1:7687"
      user: neo4j
      password: neo4j
      exec: "match (a:vertex_label)-[r:edge_label]->(b:vertex_label) return a.neo4j-source-field, b.neo4j-target-field, r.neo4j-field-0 as neo4j-field-0, r.neo4j-field-1 as neo4j-field-1 order by id(r)"
      fields: [neo4j-field-0, neo4j-field-1]
      nebula.fields: [nebula-field-0, nebula-field-1]
      source: {
        field: a.neo4j-source-field
      }
      target: {
        field: b.neo4j-target-field
      }
      partition: 10
      batch: 1000
      check_point_path: /tmp/test
    }

    # HBase
    {
      name: edge-name-4
      type: {
        source: hbase
        sink: client
      }
      host:127.0.0.1
      port:2181
      table:hbase-table
      columnFamily:hbase-table-cloumnfamily
      fields: [hbase-column-0, hbase-column-1]
      nebula.fields:[nebula-field-0, nebula-field-1]
      source: {
        field: hbase-column-k
      }
      target: {
        field: hbase-column-h
      }
      partition: 10
      batch: 1000
    }


    # Pulsar
    {
      name: edge-name-5
      type: {
        source: pulsar
        sink: client
      }
      service: "pulsar://localhost:6650"
      admin: "http://localhost:8081"
      options: {
        # choose one of "topic", "topics", "topicsPattern"
        topic: "topic1"
      }
      fields: [pulsar-field-0, pulsar-field-1, pulsar-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      source: {
        field: pulsar-field-0
        #policy: hash
      }
      target: {
        field: pulsar-field-1
      }
      ranking: pulsar-field-2
      partition: 10
      batch: 10
      interval.seconds: 10
    }

    # KAFKA
    {
      name: edge-name-6
      type: {
        source: kafka
        sink: client
      }
      service: "kafka.service.address"
      topic: "topic-name"
      fields: [kafka-field-0, kafka-field-1, kafka-field-2]
      nebula.fields: [nebula-field-0, nebula-field-1, nebula-field-2]
      source: kafka-field-0
      target: kafka-field-1
      partition: 10
      batch: 1000
      interval.seconds: 10
    }

    # MaxCompute
    {
      name: edge-name-7
      type:{
        source:maxcompute
        sink:client
      }
      table:table
      project:project
      odpsUrl:"http://service.cn-hangzhou.maxcompute.aliyun.com/api"
      tunnelUrl:"http://dt.cn-hangzhou.maxcompute.aliyun.com"
      accessKeyId:xxx
      accessKeySecret:xxx
      partitionSpec:"dt='partition1'"
      # maxcompute sql sentence only uses table name.
      sentence:"select * from table"
      fields:[maxcompute-field-0, maxcompute-field-1]
      nebula.fields:[nebula-field-0, nebula-field-1]
      source:{
        field: maxcompute-field-2
      }
      target:{
        field: maxcompute-field-3
      }
      partition:10
      batch:10
    }

    # ClickHouse
   {
      name: edge-name-7
      type: {
        source: clickhouse
        sink: client
      }
      url:"jdbc:clickhouse://127.0.0.1:8123/database"
      user:"user"
      password:"clickhouse"
      numPartition:"5"
      sentence:"select * from table"
      fields: [clickhouse-field-2]
      nebula.fields: [nebula-field-2]
      source: {
        field:clickhouse-field-0
        #policy:hash
      }
      target: {
        field:clickhouse-field-1
        #policy:hash
      }
      batch: 256
      partition: 32
    }

  ]
}