golang高性能rocksdb参数配置

golang高性能rocksdb参数配置

import (
	"errors"
	"log"
	"monorepo/service/autotable/comm/grocksdb"
	"strings"

	"github.com/sirupsen/logrus"
)

func OpenDB(DB_PATH string) (*grocksdb.DB, error) {
	options := grocksdb.NewDefaultOptions()
	options.SetCreateIfMissing(true)
	options.SetMaxFileOpeningThreads(-1)
	options.SetLevelCompactionDynamicLevelBytes(true)

	env := grocksdb.NewDefaultEnv()
	options.SetEnv(env)

	readOptions := grocksdb.NewDefaultReadOptions()
	readOptions.SetFillCache(true)

	options.SetCreateIfMissing(true)
	options.EnableStatistics()
	// memtable
	options.SetWriteBufferSize(128 * 1024 * 1024) // 128M
	env.SetBackgroundThreads(10)
	env.SetHighPriorityBackgroundThreads(1)
	options.SetMaxBackgroundFlushes(10)
	options.SetMaxWriteBufferNumber(16) // total memtable 2G

	// L0
	options.SetLevel0FileNumCompactionTrigger(8) // L0 size: 8 * 128M = 1G
	options.SetLevel0SlowdownWritesTrigger(40)   // slow down when L0 reach 5G
	options.SetLevel0StopWritesTrigger(72)       // stop when L0 reach 9G

	// L1
	options.SetTargetFileSizeBase(128 * 1024 * 1024)    // 128M, same to L0
	options.SetMaxBytesForLevelBase(1024 * 1024 * 1024) // 8 sst in L1, same number and size as L0
	// compaction
	options.SetMaxBackgroundCompactions(40)
	options.SetMaxSubcompactions(4)

	// level num
	options.SetNumLevels(4) // L0: 1G, L1: 1G, L2: 10G, L3: 100G

	// read performance
	options.SetAdviseRandomOnOpen(false) // 开启预读取

	blockBasedTableOptions := grocksdb.NewDefaultBlockBasedTableOptions()
	blockBasedTableOptions.SetBlockCache(grocksdb.NewLRUCache(1024 * 1024 * 1024)) // 1GB
	blockBasedTableOptions.SetCacheIndexAndFilterBlocks(true)
	blockBasedTableOptions.SetPinL0FilterAndIndexBlocksInCache(true)

	options.SetBlockBasedTableFactory(blockBasedTableOptions)

	db, err := grocksdb.OpenDb(options, DB_PATH)

	if err != nil {
		log.Fatalln("OPEN DB error", db, err)
		db.Close()
		return nil, errors.New("fail to open db")
	} else {
		log.Println("OPEN DB success", db)
	}
	return db, nil
}