前言
最近需要对ES数据进行分析和查询， 之前因为在入门ES时没有好好做笔记和整理 。
https://printlove.cn/tools/sql2es/
可以把基本的sql查询条件 转成es查询条件 ， 适当的使用的可以增加自己写es的效果
一般在写java ES查询时， 习惯先用 kibana 或者postman 构建ES 查询json,
其实两个工具各有优势，
两个工具都还不错， 个人偏向于postman 一点，因为注释 调试起来比较方便。
按照json的结构，可以轻松的写出java代码 ，根据query key的名称，使用QueryBuilders 构建 , 根据aggs （aggregations 的简称） key的名称,使用AggregationBuilders构建 ， 示例如下
{
"query": {
"bool": {
"must": [
{
// 时间范围
"range": {
"create_at": {
"from": "2023-09-01",
"to": "2023-09-12"
}
}
},
// 分值
{
"exists": {
"field": "score"
}
}
]
}
},
"aggs": {
"name_group": {
// 根据name分组 ,根据每组最大score排序
"terms": {
"field": "name",
"order": {
"max_score": "desc"
}
},
"aggs": {
// 最大分值
"max_score": {
"max": {
"field": "score"
}
}
}
}
}
}
// query --> java
BoolQueryBuilder conditionsBoolQuery =QueryBuilders.boolQuery()
.must(QueryBuilders.rangeQuery("create_at").from(query.getStartDateTimestamp()).to(query.getEndDateTimestamp()))
.must(QueryBuilders.existsQuery("location.county_code"));
// agg -->java
AggregationBuilder aggGroup = AggregationBuilders
.terms("name_group").field("name").order(BucketOrder.aggregation("max_score", false))
.subAggregation(AggregationBuilders.max("max_score").field("score"));
创建请求即可
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(conditionsBoolQuery).aggregation(aggGroup);
当使用RestHighLevelClient 进行es请求时 ， 关心转换 的searchSourceBuilder 入参对不对 ？ searchResponse 出参对不对？ 那么需要对es对象进行json序列化 。
当你查看es源码时 ，大部分es对象都是继承自ToXContent 对象， 而XContentBuilder 支持ToXContent 对象的自定义转换 。所以封装esHelper 方便查看调用RestHighLevelClient 入参和出参。
@Slf4j
@Component
public class EsHelper {
@Resource
private RestHighLevelClient client;
private ObjectMapper mapper;
@Value("${eshelper.debug:false}")
private boolean debug;
@PostConstruct
public void init() {
mapper = new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
// 下划线转驼峰
mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE);
}
/**
* 获取查询数量
*
* @param queryBuilder 查询器
* @param indexes 索引目录
*/
public long count(QueryBuilder queryBuilder, String... indexes) throws IOException {
CountRequest countRequest = new CountRequest(indexes);
countRequest.query(queryBuilder);
client.count(countRequest, RequestOptions.DEFAULT);
CountResponse countResponse = client.count(countRequest, RequestOptions.DEFAULT);
return countResponse.getCount();
}
/**
* 获取查询列表 List<T>
*/
public <T> List<T> listHits(Class<T> entityClass, SearchSourceBuilder sourceBuilder, String... indexes) throws IOException {
return listHits(mapper.getTypeFactory().constructType(entityClass), sourceBuilder, indexes);
}
/**
* 获取查询列表 List<Map<String, Object>>
*/
public List<Map<String, Object>> listHits(SearchSourceBuilder sourceBuilder, String... indexes) throws IOException {
return listHits(mapper.getTypeFactory().constructParametricType(Map.class, String.class, Object.class), sourceBuilder, indexes);
}
/**
* 从Hits 获取结果
*
* @param javaType 动态java类型
* @param sourceBuilder 查询构建器
* @param indexes 引目录
* @throws IOException
*/
private <T> List<T> listHits(JavaType javaType, SearchSourceBuilder sourceBuilder, String... indexes) throws IOException {
Map<String, List<T>> resultMap = new HashMap<>();
responseFilter(searchResponse -> {
SearchHit[] hits = searchResponse.getHits().getHits();
resultMap.put("result", hits2List(javaType, hits));
}, sourceBuilder, indexes);
return resultMap.getOrDefault("result", null);
}
/**
* 转换hits为java List对象
*/
public <T> List<T> hits2List(JavaType javaType, SearchHit[] hits) {
List<T> resultList = new ArrayList<>();
for (SearchHit hit : hits) {
try {
Map<String, Object> source = hit.getSourceAsMap();
source.put("id", hit.getId());
T result = mapper.readValue(mapper.writeValueAsString(source), javaType);
resultList.add(result);
} catch (Exception e) {
log.error("hit 转换对象异常!", e);
}
}
return resultList;
}
public <T> T toBean(String str, Class<T> entityClass) throws JsonProcessingException {
return mapper.readValue(str, mapper.constructType(entityClass));
}
/**
* @param responseConsumer response消费者
* @param sourceBuilder 查询构建器
* @param indexes 索引
*/
public void responseFilter(ResponseConsumer responseConsumer, SearchSourceBuilder sourceBuilder, String... indexes) throws IOException {
SearchRequest searchRequest = new SearchRequest(indexes);
searchRequest.source(sourceBuilder);
if (debug) {
log.info("ES Request json==>{}", toStringEsObject(sourceBuilder, true));
}
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
if (debug) {
log.info("ES Response json==>{} ", toStringEsObject(searchResponse, false));
}
responseConsumer.accept(searchResponse);
}
/**
* 转换es对象 为json文本
*/
protected String toStringEsObject(ToXContent xContent, boolean prettyPrint) {
XContentBuilder xContentBuilder = null;
try {
xContentBuilder = XContentBuilder.builder(XContentType.JSON.xContent());
xContent.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
return XContentHelper.convertToJson(BytesReference.bytes(xContentBuilder), prettyPrint, prettyPrint, XContentType.JSON);
} catch (IOException e) {
log.error("ES对象转json字符串失败: !!!!", e);
}
return null;
}
@FunctionalInterface
public interface ResponseConsumer {
public void accept(SearchResponse searchResponse);
}
}
ES Request json 输出SearchRequest json ，可以复制json对象至postman或 kibana 查看效果是不是正确
ES Response json输出SearchResponse json ，可以根据json 对象检查结果是否正确
